{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "Q39CIF2h3RYB",
        "2RjOSbFKzGzq",
        "Ehn-BO77Oy1h",
        "5xwO6drGPPMX",
        "CPSbGA8kPbJ2",
        "qo4pfCgzRLHh",
        "xCmgJ8K0Piwt",
        "HrAmUcMGbE0n",
        "B8874YbMc4O1"
      ],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "c32b943cf86d4922943a5b6f5c500620": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9db2e45e251e49869493b530eefa40b1",
              "IPY_MODEL_8aa3f8cc63c7436daf32ddd86d01b57d",
              "IPY_MODEL_10e6de049a1e4fd7a11f19707d52e4c7"
            ],
            "layout": "IPY_MODEL_39aab91053164a8d920586d38396ee4a"
          }
        },
        "9db2e45e251e49869493b530eefa40b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8b3295e6a508426891b613f7c2737c46",
            "placeholder": "​",
            "style": "IPY_MODEL_b0ac20d9e7744916aa13e9d3f249f02c",
            "value": "100%"
          }
        },
        "8aa3f8cc63c7436daf32ddd86d01b57d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_70c79bb109d14232b96dda8e792865d4",
            "max": 200,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9f99fd8bdbe441f58b055a9c2061589d",
            "value": 200
          }
        },
        "10e6de049a1e4fd7a11f19707d52e4c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_30ae43d1f0164765879ebb40abf73449",
            "placeholder": "​",
            "style": "IPY_MODEL_a1fcf30a878a412f9c8d897faa5975ce",
            "value": " 200/200 [05:38&lt;00:00,  1.12it/s]"
          }
        },
        "39aab91053164a8d920586d38396ee4a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8b3295e6a508426891b613f7c2737c46": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b0ac20d9e7744916aa13e9d3f249f02c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "70c79bb109d14232b96dda8e792865d4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9f99fd8bdbe441f58b055a9c2061589d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "30ae43d1f0164765879ebb40abf73449": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a1fcf30a878a412f9c8d897faa5975ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e7f1e93b10da4b26b6b19d101ba845c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4ef26b63cfb44138967e1048c5fd14a7",
              "IPY_MODEL_63d64ac2955e4ecc9f49d07c5e818bd5",
              "IPY_MODEL_6d87ca5fe84b421b9a1f4ff656fa433e"
            ],
            "layout": "IPY_MODEL_964848b1884a4466adfa773c01130a3b"
          }
        },
        "4ef26b63cfb44138967e1048c5fd14a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_780d0761e45141ebb7d1e914203fe754",
            "placeholder": "​",
            "style": "IPY_MODEL_7ef99abc89e942c8ba2f6d43a8aed7f6",
            "value": "100%"
          }
        },
        "63d64ac2955e4ecc9f49d07c5e818bd5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b336a4c41c4541b6936272aa66434fb5",
            "max": 100,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_87d3b46594294471a0fb8d054436cd67",
            "value": 100
          }
        },
        "6d87ca5fe84b421b9a1f4ff656fa433e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_aeed08bb372349889ce41c60c64cc592",
            "placeholder": "​",
            "style": "IPY_MODEL_08ed03a1cb1c42e59324d004b6447321",
            "value": " 100/100 [02:40&lt;00:00,  1.64s/it]"
          }
        },
        "964848b1884a4466adfa773c01130a3b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "780d0761e45141ebb7d1e914203fe754": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7ef99abc89e942c8ba2f6d43a8aed7f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b336a4c41c4541b6936272aa66434fb5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "87d3b46594294471a0fb8d054436cd67": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "aeed08bb372349889ce41c60c64cc592": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "08ed03a1cb1c42e59324d004b6447321": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "070b174149744dda86c66d861a892837": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_aa98a999d6ab43afa3a7ea8e6ff9291e",
              "IPY_MODEL_80c0a10f43c54c40ba033b5214efcdc9",
              "IPY_MODEL_f104258356384acbbd9100669474bb9a"
            ],
            "layout": "IPY_MODEL_cd61691c9cf14eea9075a30b3e6f66ce"
          }
        },
        "aa98a999d6ab43afa3a7ea8e6ff9291e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_096205756a364ce086b24e75fbb77459",
            "placeholder": "​",
            "style": "IPY_MODEL_93c5a5bdb0f24d76988631241e5a78e0",
            "value": "100%"
          }
        },
        "80c0a10f43c54c40ba033b5214efcdc9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d817411bfd6c471ba2237a55cac5d47d",
            "max": 200,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ac85b66dd88046f3a7b9c145cf4445af",
            "value": 200
          }
        },
        "f104258356384acbbd9100669474bb9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7a1621d311544df4826a20c2d02b35b2",
            "placeholder": "​",
            "style": "IPY_MODEL_2803f8c2b15348d1a54d6c8eac29eed3",
            "value": " 200/200 [01:10&lt;00:00,  2.12it/s]"
          }
        },
        "cd61691c9cf14eea9075a30b3e6f66ce": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "096205756a364ce086b24e75fbb77459": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "93c5a5bdb0f24d76988631241e5a78e0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d817411bfd6c471ba2237a55cac5d47d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ac85b66dd88046f3a7b9c145cf4445af": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7a1621d311544df4826a20c2d02b35b2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2803f8c2b15348d1a54d6c8eac29eed3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "52de909e363a475e8de11e0254f0e6c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_de5fd14fbeba439f929f12d3a5f8d834",
              "IPY_MODEL_59a70ef1c554449c8ae1956b7f73d5f2",
              "IPY_MODEL_4b8c5bf518084b90b38095e6dbbe7b57"
            ],
            "layout": "IPY_MODEL_dc7ea97773a94170a96a56c0c415f4d4"
          }
        },
        "de5fd14fbeba439f929f12d3a5f8d834": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ef452f90672b471f941119c418e74754",
            "placeholder": "​",
            "style": "IPY_MODEL_64fd470c744f41409dee05d7e3c6b0e7",
            "value": "100%"
          }
        },
        "59a70ef1c554449c8ae1956b7f73d5f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cdfc6438f2ac4f83a3a1dfed3cc4439c",
            "max": 100,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5781329ba3844daca20e076e07d354f3",
            "value": 100
          }
        },
        "4b8c5bf518084b90b38095e6dbbe7b57": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a8e0f4f3e28c4a7994769c56ff0aaa02",
            "placeholder": "​",
            "style": "IPY_MODEL_d5eba784fd634c5e9aefd84855119a63",
            "value": " 100/100 [00:34&lt;00:00,  2.74it/s]"
          }
        },
        "dc7ea97773a94170a96a56c0c415f4d4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ef452f90672b471f941119c418e74754": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "64fd470c744f41409dee05d7e3c6b0e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cdfc6438f2ac4f83a3a1dfed3cc4439c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5781329ba3844daca20e076e07d354f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a8e0f4f3e28c4a7994769c56ff0aaa02": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d5eba784fd634c5e9aefd84855119a63": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "##### Copyright 2023 Google LLC. SPDX-License-Identifier: Apache-2.0"
      ],
      "metadata": {
        "id": "OmYi8qM4WebZ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Copyright 2023 Google LLC. SPDX-License-Identifier: Apache-2.0\n",
        "\n",
        "Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at\n",
        "\n",
        "https://www.apache.org/licenses/LICENSE-2.0\n",
        "\n",
        "Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License."
      ],
      "metadata": {
        "id": "kHO2w5zSWbsf"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Robots That Ask For Help: Uncertainty Alignment for Large Language Model Planners** Mobile Manipulation\n",
        "\n",
        "[KnowNo](https://robot-help.github.io) is a framework for measuring and aligning the uncertainty of LLM-based planners, such that they know when they don't know, and ask for help when needed. KnowNo builds on the theory of conformal prediction to provide statistical guarantees on task completion while minimizing human help.\n",
        "\n",
        "This colab runs an example of KnowNo for mobile manipulation (example scenario shown below)\n",
        "*   Environment: an orange, an apple, and a Sprite\n",
        "*   User instruction: Put the fruit in the top drawer.\n",
        "*   Ambiguity: fruit is under-specified (apple or orange). User meant orange.\n",
        "\n",
        "<img src=\"https://robot-help.github.io/img/sample-mobile-manipulation.png\" height=\"200px\">\n",
        "\n",
        "Note:\n",
        "* Instead of setting up the scenario distribution here, we will load a dataset sampled from a pre-defined scenario distribution involving the mobile robot, the same used in the experiments.\n",
        "* We use [GPT-3.5](https://arxiv.org/abs/2005.14165) (text-davinci-003) as the language model here.\n",
        "* We focus on the planning part; we do not consider object detection or low-level action execution here.\n",
        "* The calibration invoves running LLM inference for a fair number of calibration data (200). You can also skip the calibration and directly test in new scenarios at the last block based on provided calibration result."
      ],
      "metadata": {
        "id": "cHsncwrPOxZt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "openai_api_key = \"your-api-key\""
      ],
      "metadata": {
        "id": "irR8HHXQWyoD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## **Setup**"
      ],
      "metadata": {
        "id": "Q39CIF2h3RYB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown A few imports and downloading data\n",
        "!pip install -U --no-cache-dir gdown --pre\n",
        "!pip install openai tqdm\n",
        "\n",
        "import openai\n",
        "import signal\n",
        "import tqdm.notebook as tqdm\n",
        "import random\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# dataset files - small sizes\n",
        "!gdown https://drive.google.com/uc?id=1XWIeGfF08V1eR104VLDilmwhIGVk2uzk\n",
        "!gdown https://drive.google.com/uc?id=1iEIZaVbbajMXsNdrjVkOgK5rhPtfl5WI\n",
        "\n",
        "# Set OpenAI API key.\n",
        "openai.api_key = openai_api_key"
      ],
      "metadata": {
        "cellView": "form",
        "id": "QSmQIVIIarLz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown LLM API call\n",
        "class timeout:\n",
        "    def __init__(self, seconds=1, error_message='Timeout'):\n",
        "        self.seconds = seconds\n",
        "        self.error_message = error_message\n",
        "\n",
        "    def handle_timeout(self, signum, frame):\n",
        "        raise TimeoutError(self.error_message)\n",
        "\n",
        "    def __enter__(self):\n",
        "        signal.signal(signal.SIGALRM, self.handle_timeout)\n",
        "        signal.alarm(self.seconds)\n",
        "\n",
        "    def __exit__(self, type, value, traceback):\n",
        "        signal.alarm(0)\n",
        "\n",
        "# OpenAI only supports up to five tokens (logprobs argument) for getting the likelihood.\n",
        "# Thus we use the logit_bias argument to force LLM only consdering the five option\n",
        "# tokens: A, B, C, D, E\n",
        "def lm(prompt,\n",
        "       max_tokens=256,\n",
        "       temperature=0,\n",
        "       logprobs=None,\n",
        "       stop_seq=None,\n",
        "       logit_bias={\n",
        "          317: 100.0,   #  A (with space at front)\n",
        "          347: 100.0,   #  B (with space at front)\n",
        "          327: 100.0,   #  C (with space at front)\n",
        "          360: 100.0,   #  D (with space at front)\n",
        "          412: 100.0,   #  E (with space at front)\n",
        "      },\n",
        "       timeout_seconds=20):\n",
        "  max_attempts = 5\n",
        "  for _ in range(max_attempts):\n",
        "      try:\n",
        "          with timeout(seconds=timeout_seconds):\n",
        "              response = openai.Completion.create(\n",
        "                  model='text-davinci-003',\n",
        "                  prompt=prompt,\n",
        "                  max_tokens=max_tokens,\n",
        "                  temperature=temperature,\n",
        "                  logprobs=logprobs,\n",
        "                  logit_bias=logit_bias,\n",
        "                  stop=list(stop_seq) if stop_seq is not None else None,\n",
        "              )\n",
        "          break\n",
        "      except:\n",
        "          print('Timeout, retrying...')\n",
        "          pass\n",
        "  return response, response[\"choices\"][0][\"text\"].strip()"
      ],
      "metadata": {
        "id": "DpR4dgevMMsa",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## **Load Pre-generated Dataset**\n",
        "KnowNo assumes there exists a distribution of scenarios that the robot may encounter. A scenario refers to an environment with objects, a (potentially ambiguous) language instruction, and a goal. Here we provide a dataset of 300 scenarios sampled from a pre-defined scenario distribution, including the corresponding prompts for the LLM.\n",
        "\n",
        "We will also split the dataset into a calibration set of size 200 and a test set of size 100."
      ],
      "metadata": {
        "id": "2RjOSbFKzGzq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@title\n",
        "# Load the 300 scenarios\n",
        "scenario_info_path = '/content/metabot-tasks-info.txt'\n",
        "with open(scenario_info_path, 'r') as f:\n",
        "  scenario_info_text = f.read()\n",
        "scenario_info_text = scenario_info_text.split('\\n\\n')\n",
        "print('Loaded scenario info from ' + scenario_info_path)"
      ],
      "metadata": {
        "id": "KIj6ncIPAM1a",
        "outputId": "3d1793aa-4f11-4f5b-9732-65e5bdc0b468",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loaded scenario info from /content/metabot-tasks-info.txt\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@title\n",
        "# Print a scenario\n",
        "print('Sample scenario:\\n')\n",
        "print(scenario_info_text[0].split('\\n',1)[1])  # remove the printed index"
      ],
      "metadata": {
        "collapsed": true,
        "id": "MzUjuVWD2OE4",
        "outputId": "a242a647-1ca0-41d3-ee8c-a2ae748d8158",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sample scenario:\n",
            "\n",
            "Scene: a Coke, a bottled unsweetened tea, and a Sprite\n",
            "Task: Bring me a flavored drink.\n",
            "User intent (object): Coke, bottled unsweetened tea, Sprite\n",
            "User intent (location): pick-up\n",
            "Scene objects: Coke, bottled unsweetened tea, Sprite\n",
            "Task category: creative_multilabel_task\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@title\n",
        "# Load the corresponding prompts for multiple choice generation\n",
        "mc_gen_prompt_path = '/content/metabot-mc-gen-prompt.txt'\n",
        "with open(mc_gen_prompt_path, 'r') as f:\n",
        "  mc_gen_prompt_all = f.read().split('--0000--')\n",
        "print('Loaded multiple choice generation prompts from ' + mc_gen_prompt_path)"
      ],
      "metadata": {
        "id": "QcgT1Hmh5J4B",
        "outputId": "b912ecad-d90b-46cf-f9f5-2832ef9c51e8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "cellView": "form",
        "collapsed": true
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loaded multiple choice generation prompts from /content/metabot-mc-gen-prompt.txt\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@title\n",
        "# Print a prompt for MC generation\n",
        "print(mc_gen_prompt_all[0])"
      ],
      "metadata": {
        "collapsed": true,
        "id": "ChAQLxXi7IIm",
        "outputId": "457ab1e9-8a5a-4408-b4da-606e9bcb29c5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "We: You are a robot operating in an office kitchen. You are in front of a counter with two closed drawers, a top one and a bottom one. There is also a landfill bin, a recycling bin, and a compost bin.\n",
            "\n",
            "We: On the counter, there is an orange soda, a Pepsi, and an apple.\n",
            "We: Put that drink in the top drawer.\n",
            "You:\n",
            "A) open the top drawer and put the orange soda in it\n",
            "B) open the bottom drawer and put the Pepsi in it\n",
            "C) open the bottom drawer and put the orange soda in it\n",
            "D) open the top drawer and put the Pepsi in it\n",
            "\n",
            "We: On the counter, there is an energy bar, a banana, and a microwave.\n",
            "We: Put the snack next to the microwave.\n",
            "You:\n",
            "A) pick up the energy bar and put it next to the microwave\n",
            "B) pick up the banana and put it next to the energy bar\n",
            "C) pick up the banana and put it next to the microwave\n",
            "D) pick up the energy bar and put it next to the banana\n",
            "\n",
            "We: On the counter, there is a Coke, a Sprite, and a sponge.\n",
            "We: Can you dispose of the can? It should have expired.\n",
            "You:\n",
            "A) pick up the sponge and put it in the landfill bin\n",
            "B) pick up the Coke and put it in the recycling bin\n",
            "C) pick up the Sprite and put it in the recycling bin\n",
            "D) pick up the Coke and put it in the landfill bin\n",
            "\n",
            "We: On the counter, there is a bottled water, a bag of jalapeno chips, and a bag of rice chips.\n",
            "We: I would like a bag of chips.\n",
            "You:\n",
            "A) pick up the bottled water\n",
            "B) pick up the jalapeno chips\n",
            "C) pick up the kettle chips\n",
            "D) pick up the rice chips\n",
            "\n",
            "We: On the counter, there is a Coke, a bottled unsweetened tea, and a Sprite. \n",
            "We: Bring me a flavored drink.\n",
            "You:\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@title\n",
        "# Split data into calibration set and test set\n",
        "calibration_set = []\n",
        "num_calibration_data = 200\n",
        "num_test_data = 100\n",
        "for i in range(num_calibration_data):\n",
        "  calibration_set.append({\n",
        "      'info': scenario_info_text[i],\n",
        "      'mc_gen_prompt': mc_gen_prompt_all[i],\n",
        "  })\n",
        "test_set = []\n",
        "for i in range(num_calibration_data, 300):\n",
        "  test_set.append({\n",
        "      'info': scenario_info_text[i],\n",
        "      'mc_gen_prompt': mc_gen_prompt_all[i],\n",
        "  })"
      ],
      "metadata": {
        "id": "cUaEe4j97RDz",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "## **Multiple Choice Question Answering**\n",
        "For each scenario, KnowNo first applies few-shot prompting to generate plausible options to take, and then queries the likelihood of predicting the choice token (A, B, C, D, E). These likelihood values reveal the LLM uncertainty."
      ],
      "metadata": {
        "id": "2ln3p3jE0taA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown Print an example of task and prompt for MC generation\n",
        "print('Sample task:\\n')\n",
        "print(calibration_set[0]['info'].split('\\n',1)[1])  # remove the printed index\n",
        "print('\\n-------------\\nPrompt for MC generation:\\n')\n",
        "print(calibration_set[0]['mc_gen_prompt'])"
      ],
      "metadata": {
        "id": "qczjVeUrg0NI",
        "collapsed": true,
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "cellView": "form",
        "outputId": "200c2dbb-3a85-4d34-b04a-fa57661da802"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sample task:\n",
            "\n",
            "Scene: a Coke, a bottled unsweetened tea, and a Sprite\n",
            "Task: Bring me a flavored drink.\n",
            "User intent (object): Coke, bottled unsweetened tea, Sprite\n",
            "User intent (location): pick-up\n",
            "Scene objects: Coke, bottled unsweetened tea, Sprite\n",
            "Task category: creative_multilabel_task\n",
            "\n",
            "-------------\n",
            "Prompt for MC generation:\n",
            "\n",
            "We: You are a robot operating in an office kitchen. You are in front of a counter with two closed drawers, a top one and a bottom one. There is also a landfill bin, a recycling bin, and a compost bin.\n",
            "\n",
            "We: On the counter, there is an orange soda, a Pepsi, and an apple.\n",
            "We: Put that drink in the top drawer.\n",
            "You:\n",
            "A) open the top drawer and put the orange soda in it\n",
            "B) open the bottom drawer and put the Pepsi in it\n",
            "C) open the bottom drawer and put the orange soda in it\n",
            "D) open the top drawer and put the Pepsi in it\n",
            "\n",
            "We: On the counter, there is an energy bar, a banana, and a microwave.\n",
            "We: Put the snack next to the microwave.\n",
            "You:\n",
            "A) pick up the energy bar and put it next to the microwave\n",
            "B) pick up the banana and put it next to the energy bar\n",
            "C) pick up the banana and put it next to the microwave\n",
            "D) pick up the energy bar and put it next to the banana\n",
            "\n",
            "We: On the counter, there is a Coke, a Sprite, and a sponge.\n",
            "We: Can you dispose of the can? It should have expired.\n",
            "You:\n",
            "A) pick up the sponge and put it in the landfill bin\n",
            "B) pick up the Coke and put it in the recycling bin\n",
            "C) pick up the Sprite and put it in the recycling bin\n",
            "D) pick up the Coke and put it in the landfill bin\n",
            "\n",
            "We: On the counter, there is a bottled water, a bag of jalapeno chips, and a bag of rice chips.\n",
            "We: I would like a bag of chips.\n",
            "You:\n",
            "A) pick up the bottled water\n",
            "B) pick up the jalapeno chips\n",
            "C) pick up the kettle chips\n",
            "D) pick up the rice chips\n",
            "\n",
            "We: On the counter, there is a Coke, a bottled unsweetened tea, and a Sprite. \n",
            "We: Bring me a flavored drink.\n",
            "You:\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown Run MC generation (this may take a few minutes depending on API traffic)\n",
        "import tqdm.notebook as tqdm\n",
        "\n",
        "print('Running calibration set...printing first five results...')\n",
        "for i in tqdm.trange(num_calibration_data):\n",
        "  calibration_data = calibration_set[i]\n",
        "  prompt = calibration_data['mc_gen_prompt']\n",
        "  _, text = lm(prompt, logit_bias={})\n",
        "  text = text.strip()\n",
        "  # print a few results\n",
        "  if i < 5:\n",
        "    print(text)\n",
        "    print('----')\n",
        "  calibration_data['mc_gen_raw'] = text\n",
        "  calibration_set[i] = calibration_data\n",
        "\n",
        "print('Running test set...printing first five results...')\n",
        "for i in tqdm.trange(num_test_data):\n",
        "  test_data = test_set[i]\n",
        "  prompt = test_data['mc_gen_prompt']\n",
        "  _, text = lm(prompt, logit_bias={})\n",
        "  text = text.strip()\n",
        "  test_data['mc_gen_raw'] = text\n",
        "  test_set[i] = test_data"
      ],
      "metadata": {
        "id": "iuddtTvdEcZA",
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 550,
          "referenced_widgets": [
            "c32b943cf86d4922943a5b6f5c500620",
            "9db2e45e251e49869493b530eefa40b1",
            "8aa3f8cc63c7436daf32ddd86d01b57d",
            "10e6de049a1e4fd7a11f19707d52e4c7",
            "39aab91053164a8d920586d38396ee4a",
            "8b3295e6a508426891b613f7c2737c46",
            "b0ac20d9e7744916aa13e9d3f249f02c",
            "70c79bb109d14232b96dda8e792865d4",
            "9f99fd8bdbe441f58b055a9c2061589d",
            "30ae43d1f0164765879ebb40abf73449",
            "a1fcf30a878a412f9c8d897faa5975ce",
            "e7f1e93b10da4b26b6b19d101ba845c6",
            "4ef26b63cfb44138967e1048c5fd14a7",
            "63d64ac2955e4ecc9f49d07c5e818bd5",
            "6d87ca5fe84b421b9a1f4ff656fa433e",
            "964848b1884a4466adfa773c01130a3b",
            "780d0761e45141ebb7d1e914203fe754",
            "7ef99abc89e942c8ba2f6d43a8aed7f6",
            "b336a4c41c4541b6936272aa66434fb5",
            "87d3b46594294471a0fb8d054436cd67",
            "aeed08bb372349889ce41c60c64cc592",
            "08ed03a1cb1c42e59324d004b6447321"
          ]
        },
        "outputId": "8fdcde0d-9162-4bc1-b559-f7368982ff63"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Running calibration set...printing first five results...\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/200 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c32b943cf86d4922943a5b6f5c500620"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "A) pick up the Coke\n",
            "B) pick up the bottled unsweetened tea\n",
            "C) pick up the Sprite\n",
            "D) pick up the bottled sweetened tea\n",
            "----\n",
            "A) pick up the Pepsi\n",
            "B) pick up the Coke\n",
            "C) pick up the RedBull\n",
            "D) pick up the bottled water\n",
            "----\n",
            "A) pick up the orange soda and put it in the landfill bin\n",
            "B) pick up the orange and put it in the landfill bin\n",
            "C) pick up the apple and put it in the landfill bin\n",
            "D) pick up the orange soda and put it in the recycling bin\n",
            "----\n",
            "A) open the top drawer and put the Coke in it\n",
            "B) open the bottom drawer and put the Coke in it\n",
            "C) open the top drawer and put the energy bar in it\n",
            "D) open the bottom drawer and put the energy bar in it\n",
            "----\n",
            "A) pick up the RedBull and put it in the landfill bin\n",
            "B) pick up the bottled unsweetened tea and put it in the landfill bin\n",
            "C) pick up the bottled water and put it in the recycling bin\n",
            "D) pick up the RedBull and put it in the recycling bin\n",
            "----\n",
            "Running test set...printing first five results...\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/100 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e7f1e93b10da4b26b6b19d101ba845c6"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown Post-process the generated MC and get prompt for querying likelihood\n",
        "\n",
        "mc_score_background_prompt = \"You are a robot operating in an office kitchen. \"\\\n",
        "\"You are in front of a counter with two closed drawers, a top one and a middle \"\\\n",
        "\"one. There is also a landfill bin, a recycling bin, and a compost bin.\"\n",
        "\n",
        "def process_mc_raw(mc_raw, add_mc='an option not listed here'):\n",
        "  mc_all = mc_raw.split('\\n')\n",
        "\n",
        "  mc_processed_all = []\n",
        "  for mc in mc_all:\n",
        "      mc = mc.strip()\n",
        "\n",
        "      # skip nonsense\n",
        "      if len(mc) < 5 or mc[0] not in [\n",
        "          'a', 'b', 'c', 'd', 'A', 'B', 'C', 'D', '1', '2', '3', '4'\n",
        "      ]:\n",
        "          continue\n",
        "      mc = mc[2:]  # remove a), b), ...\n",
        "      mc = mc.strip().lower().split('.')[0]\n",
        "      mc_processed_all.append(mc)\n",
        "  if len(mc_processed_all) < 4:\n",
        "      raise 'Cannot extract four options from the raw output.'\n",
        "\n",
        "  # Check if any repeated option - use do nothing as substitue\n",
        "  mc_processed_all = list(set(mc_processed_all))\n",
        "  if len(mc_processed_all) < 4:\n",
        "      num_need = 4 - len(mc_processed_all)\n",
        "      for _ in range(num_need):\n",
        "          mc_processed_all.append('do nothing')\n",
        "  prefix_all = ['A) ', 'B) ', 'C) ', 'D) ']\n",
        "  if add_mc is not None:\n",
        "      mc_processed_all.append(add_mc)\n",
        "      prefix_all.append('E) ')\n",
        "  random.shuffle(mc_processed_all)\n",
        "\n",
        "  # get full string\n",
        "  mc_prompt = ''\n",
        "  for mc_ind, (prefix, mc) in enumerate(zip(prefix_all, mc_processed_all)):\n",
        "      mc_prompt += prefix + mc\n",
        "      if mc_ind < len(mc_processed_all) - 1:\n",
        "          mc_prompt += '\\n'\n",
        "  add_mc_prefix = prefix_all[mc_processed_all.index(add_mc)][0]\n",
        "  return mc_prompt, mc_processed_all, add_mc_prefix\n",
        "\n",
        "# process\n",
        "for dataset in [calibration_set, test_set]:\n",
        "  for i in range(len(dataset)):\n",
        "    mc_gen_raw = dataset[i]['mc_gen_raw'].strip()\n",
        "    mc_gen_full, mc_gen_all, add_mc_prefix = process_mc_raw(mc_gen_raw)\n",
        "\n",
        "    # get the part of the current scenario from the previous prompt\n",
        "    cur_scenario_prompt = dataset[i]['mc_gen_prompt'].split('\\n\\n')[-1]\n",
        "\n",
        "    # get new prompt\n",
        "    mc_score_prompt = mc_score_background_prompt + '\\n\\n' + cur_scenario_prompt + '\\n' + mc_gen_full\n",
        "    mc_score_prompt += \"\\nWe: Which option is correct? Answer with a single letter.\"\n",
        "    mc_score_prompt += \"\\nYou:\"\n",
        "    dataset[i]['mc_score_prompt'] = mc_score_prompt\n",
        "\n",
        "    # save other data\n",
        "    dataset[i]['mc_gen_full'] = mc_gen_full\n",
        "    dataset[i]['mc_gen_all'] = mc_gen_all\n",
        "    dataset[i]['add_mc_prefix'] = add_mc_prefix"
      ],
      "metadata": {
        "id": "COcmIMfUUoIQ",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown Print an example of prompt for querying likelihood\n",
        "print(calibration_set[0]['mc_score_prompt'])"
      ],
      "metadata": {
        "id": "GGwGheSvRQEW",
        "outputId": "e944fa6f-550d-4541-d125-f5c4a81f3f90",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "You are a robot operating in an office kitchen. You are in front of a counter with two closed drawers, a top one and a middle one. There is also a landfill bin, a recycling bin, and a compost bin.\n",
            "\n",
            "We: On the counter, there is a Coke, a bottled unsweetened tea, and a Sprite. \n",
            "We: Bring me a flavored drink.\n",
            "You:\n",
            "A) pick up the coke\n",
            "B) an option not listed here\n",
            "C) pick up the bottled sweetened tea\n",
            "D) pick up the bottled unsweetened tea\n",
            "E) pick up the sprite\n",
            "We: Which option is correct? Answer with a single letter.\n",
            "You:\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown Query the likelihood of choice tokens (this may take a few minutes depending on API traffic)\n",
        "for i, dataset in enumerate([calibration_set, test_set]):\n",
        "  if i == 0:\n",
        "    print('Running calibration set...printing first five results...')\n",
        "  else:\n",
        "    print('Running test set...printing first five results...')\n",
        "  for i in tqdm.trange(len(dataset)):\n",
        "    prompt = dataset[i]['mc_score_prompt']\n",
        "\n",
        "    # call LLM API\n",
        "    mc_score_response, _ = lm(prompt, max_tokens=1, logprobs=5)\n",
        "    top_logprobs_full = mc_score_response[\"choices\"][0][\"logprobs\"][\"top_logprobs\"][0]\n",
        "    top_tokens = [token.strip() for token in top_logprobs_full.keys()]\n",
        "    top_logprobs = [value for value in top_logprobs_full.values()]\n",
        "    if i < 5:\n",
        "      print(top_tokens, top_logprobs)\n",
        "    dataset[i]['top_logprobs_full'] = top_logprobs_full\n",
        "    dataset[i]['top_tokens'] = top_tokens\n",
        "    dataset[i]['top_logprobs'] = top_logprobs"
      ],
      "metadata": {
        "colab": {
          "height": 289,
          "referenced_widgets": [
            "070b174149744dda86c66d861a892837",
            "aa98a999d6ab43afa3a7ea8e6ff9291e",
            "80c0a10f43c54c40ba033b5214efcdc9",
            "f104258356384acbbd9100669474bb9a",
            "cd61691c9cf14eea9075a30b3e6f66ce",
            "096205756a364ce086b24e75fbb77459",
            "93c5a5bdb0f24d76988631241e5a78e0",
            "d817411bfd6c471ba2237a55cac5d47d",
            "ac85b66dd88046f3a7b9c145cf4445af",
            "7a1621d311544df4826a20c2d02b35b2",
            "2803f8c2b15348d1a54d6c8eac29eed3",
            "52de909e363a475e8de11e0254f0e6c7",
            "de5fd14fbeba439f929f12d3a5f8d834",
            "59a70ef1c554449c8ae1956b7f73d5f2",
            "4b8c5bf518084b90b38095e6dbbe7b57",
            "dc7ea97773a94170a96a56c0c415f4d4",
            "ef452f90672b471f941119c418e74754",
            "64fd470c744f41409dee05d7e3c6b0e7",
            "cdfc6438f2ac4f83a3a1dfed3cc4439c",
            "5781329ba3844daca20e076e07d354f3",
            "a8e0f4f3e28c4a7994769c56ff0aaa02",
            "d5eba784fd634c5e9aefd84855119a63"
          ],
          "base_uri": "https://localhost:8080/"
        },
        "id": "3lr-wxgwYjub",
        "outputId": "b96dd157-85b8-4f65-b76b-a67901c181cc",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Running calibration set...printing first five results...\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/200 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "070b174149744dda86c66d861a892837"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['E', 'D', 'C', 'A', 'B'] [-0.2519463, -1.5561227, -5.050118, -5.316033, -7.727799]\n",
            "['E', 'D', 'B', 'A', 'C'] [-0.002713803, -5.920858, -10.699895, -12.656659, -13.417515]\n",
            "['B', 'C', 'A', 'D', 'E'] [-0.0003345106, -8.228652, -9.603835, -15.858969, -17.2986]\n",
            "['A', 'E', 'B', 'D', 'C'] [-0.20416771, -1.6897938, -9.600461, -10.388485, -10.906881]\n",
            "['B', 'C', 'A', 'D', 'E'] [-0.009360072, -4.6867967, -9.218649, -14.434767, -14.504599]\n",
            "Running test set...printing first five results...\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/100 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "52de909e363a475e8de11e0254f0e6c7"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['E', 'C', 'A', 'B', 'D'] [-0.004876243, -5.3763742, -8.366593, -11.975548, -13.7236185]\n",
            "['D', 'A', 'B', 'E', 'C'] [-2.5344274e-05, -11.427271, -11.657274, -12.264673, -13.826197]\n",
            "['D', 'A', 'B', 'C', 'E'] [-1.5805701e-05, -11.204811, -13.664916, -13.807395, -17.985342]\n",
            "['A', 'C', 'B', 'E', 'D'] [-8.5358415e-06, -12.154228, -12.661491, -16.09182, -17.296272]\n",
            "['E', 'D', 'C', 'A', 'B'] [-0.34237248, -1.2382848, -10.517009, -12.27785, -12.517641]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## **Specify Target Success Level**\n",
        "Here you can specify the target success level, which is $1-\\epsilon$ in the paper.\n"
      ],
      "metadata": {
        "id": "E9aZi0XmcPN1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "target_success = 0.8  #@param {type: \"number\"} {form-width: \"10%\"}\n",
        "epsilon = 1-target_success\n",
        "print('Epsilon:', epsilon)"
      ],
      "metadata": {
        "cellView": "form",
        "id": "err6DW0NcTC2",
        "outputId": "588badff-5a9a-4341-8029-e1552fbd3c93",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epsilon: 0.19999999999999996\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## **Calibration with Conformal Prediction**\n",
        "We now perform calibration based on likelihood from the calibration set and the target error level $ϵ$. The result is a threshold $1-\\widehat{q}$ that at test time, if the prediction set includes all options with likelihood higher than $1-\\widehat{q}$, then the set is guaranteed to include the **true** option with $1-\\epsilon$ probability."
      ],
      "metadata": {
        "id": "QQIWiYsqYEYZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown First, for convenience, we use simple heuristics to determine if each option generated is a true option. In the pre-generated data, we defined the user intent (object to be moved, and the target location). In the paper for the Mobile Manipulation example, the true options are labelled manually.\n",
        "for dataset in [calibration_set, test_set]:\n",
        "  for data_ind, data in enumerate(dataset):\n",
        "    true_options = []\n",
        "    info = data['info']\n",
        "    mc_gen_all = data['mc_gen_all']\n",
        "\n",
        "    # extract the line started with user intent (object) from info\n",
        "    info = info.split('\\n',1)[1]  # remove index\n",
        "    true_obj = info.split('User intent (object): ')[1].split('\\n')[0].lower()\n",
        "    true_obj = true_obj.split(',')\n",
        "    true_obj = [obj.strip() for obj in true_obj]\n",
        "    true_target_loc = info.split('User intent (location): ')[1].split('\\n')[0].lower()\n",
        "    # print(true_obj, '   ', true_target_loc)\n",
        "\n",
        "    # go through all mc\n",
        "    token_all = ['A', 'B', 'C', 'D', 'E']\n",
        "    for mc_ind, mc in enumerate(mc_gen_all):\n",
        "      if 'not listed here' in mc or 'do nothing' in mc: continue\n",
        "\n",
        "      # corner case: if there is only one sponge, shrink true_obj to only sponge\n",
        "      scene_obj = info.split('Scene objects:')[1].split('\\n')[0].split(', ')\n",
        "      scene_obj = [obj.strip().lower() for obj in scene_obj]\n",
        "      for i in range(len(true_obj)):\n",
        "        if not ('clean sponge' in scene_obj and 'dirty sponge with food residue' in scene_obj) \\\n",
        "          and 'sponge' in true_obj[i]:\n",
        "          true_obj[i] = 'sponge'\n",
        "\n",
        "      # corner case: mc has both clean and dirty\n",
        "      if 'clean' in mc and 'dirty' in mc: continue\n",
        "\n",
        "      # corner case: orange and orange soda - not dealt with\n",
        "\n",
        "      if true_target_loc == 'pick-up':\n",
        "        # check if more than one scene object in the mc\n",
        "        num_obj_in_mc = 0\n",
        "        for obj in scene_obj:\n",
        "          if obj in mc.lower(): num_obj_in_mc += 1\n",
        "        if num_obj_in_mc > 1: continue\n",
        "\n",
        "        for obj in true_obj:\n",
        "          if obj in mc and 'drawer' not in mc and 'bin' not in mc and \\\n",
        "            'microwave' not in mc and 'cooktop' not in mc:\n",
        "            true_options.append(token_all[mc_ind])\n",
        "      elif 'drawer' in true_target_loc:\n",
        "        for obj in true_obj:\n",
        "          if obj in mc and true_target_loc in mc:\n",
        "            true_options.append(token_all[mc_ind])\n",
        "      elif 'recycling' in true_target_loc or 'landfill' in true_target_loc \\\n",
        "        or 'compost' in true_target_loc or 'microwave' in true_target_loc \\\n",
        "        or 'cooktop' in true_target_loc:\n",
        "        for obj in true_obj:\n",
        "          if obj in mc and true_target_loc in mc:\n",
        "            true_options.append(token_all[mc_ind])\n",
        "      else: # target location is an object\n",
        "        mc_obj_pick_up_phrase = mc.split('and')[0]\n",
        "        mc_obj_place_phrase = mc.split('and')[1]\n",
        "        for obj in true_obj:\n",
        "          if obj in mc_obj_pick_up_phrase and true_target_loc in mc_obj_place_phrase:\n",
        "            true_options.append(token_all[mc_ind])\n",
        "\n",
        "    # if none correct\n",
        "    if len(true_options) == 0:\n",
        "      true_options = [data['add_mc_prefix']]\n",
        "\n",
        "    # save\n",
        "    dataset[data_ind]['true_options'] = true_options"
      ],
      "metadata": {
        "id": "pbmHI99DXPHY",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown Then, get the non-conformity scores from the calibration set, which is 1 minus the likelihood of the **true** option, $1-f(x)_{y_\\text{true}}$. If there is more than one acceptable option for a scenario, we will use the minimum.\n",
        "def temperature_scaling(logits, temperature):\n",
        "    logits = np.array(logits)\n",
        "    logits /= temperature\n",
        "\n",
        "    # apply softmax\n",
        "    logits -= logits.max()\n",
        "    logits = logits - np.log(np.sum(np.exp(logits)))\n",
        "    smx = np.exp(logits)\n",
        "    return smx\n",
        "\n",
        "non_conformity_score = []\n",
        "token_all = ['A', 'B', 'C', 'D', 'E']\n",
        "for data in calibration_set:\n",
        "  top_logprobs = data['top_logprobs']\n",
        "  top_tokens = data['top_tokens']\n",
        "  true_options = data['true_options']\n",
        "\n",
        "  # normalize the five scores to sum of 1\n",
        "  mc_smx_all = temperature_scaling(top_logprobs, temperature=5)\n",
        "\n",
        "  # get the softmax value of true option\n",
        "  true_label_smx = [mc_smx_all[token_ind]\n",
        "                    for token_ind, token in enumerate(top_tokens)\n",
        "                    if token in true_options]\n",
        "  true_label_smx = np.max(true_label_smx)\n",
        "\n",
        "  # get non-comformity score\n",
        "  non_conformity_score.append(1 - true_label_smx)"
      ],
      "metadata": {
        "cellView": "form",
        "id": "gJkulUUPRy-d"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown Find $\\widehat{q}$ as the $\\frac{\\lceil (N+1)(1-\\epsilon) \\rceil}{N}$ quantile of the non-conformity scores, where $N$ is number of calibration data. Also plot the histogram of the scores and the quantile.\n",
        "q_level = np.ceil((num_calibration_data + 1) * (1 - epsilon)) / num_calibration_data\n",
        "qhat = np.quantile(non_conformity_score, q_level, method='higher')\n",
        "print('Quantile value qhat:', qhat)\n",
        "print('')\n",
        "\n",
        "# plot histogram and quantile\n",
        "plt.figure(figsize=(6, 3))\n",
        "plt.hist(non_conformity_score, bins=30, edgecolor='k', linewidth=1)\n",
        "plt.axvline(\n",
        "    x=qhat, linestyle='--', color='r', label='Quantile value'\n",
        ")\n",
        "plt.title(\n",
        "    'Histogram of non-comformity scores in the calibration set'\n",
        ")\n",
        "plt.xlabel('Non-comformity score')\n",
        "plt.legend(); plt.show()\n",
        "print('')\n",
        "print('A good predictor should have low non-comformity scores, concentrated at the left side of the figure')"
      ],
      "metadata": {
        "colab": {
          "height": 402,
          "base_uri": "https://localhost:8080/"
        },
        "id": "MZmTNEGQbeqx",
        "outputId": "792d1b63-f90e-489b-d5bf-b099eddee176",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Quantile value qhat: 0.928147098697616\n",
            "\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 600x300 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAggAAAE8CAYAAACl/0lFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABIq0lEQVR4nO3deVwU9f8H8NeCsIucgXLJrXijJkoeeWuIaZ6ZRwpqUYl5W5p5oCkepZRZavZFK880tcw8y9ssDzIV8cL7BEUEuffz+8Mfk8ssyMKyC+zr+Xjsg93PXO+Zz+zsm5nPZ0YhhBAgIiIieoaZsQMgIiKisocJAhEREckwQSAiIiIZJghEREQkwwSBiIiIZJggEBERkQwTBCIiIpJhgkBEREQyTBCIiIhIptwlCD4+PggLCzN2GBXe/Pnz4efnB3NzczRq1MjY4ZRrd+/eRZ8+feDk5ASFQoHo6GhjhwQA2Lt3LxQKBfbu3WvsUCoMQ27T6dOnQ6FQIDExsdSXpS9XrlyBQqHAihUrpLK89XiWQqHAiBEjDBydnKn/3hg1QVixYgUUCgWOHTumdXjbtm1Rv379Ei9n27ZtmD59eonnYyp27tyJDz74AC1btkRMTAxmz55t7JDKtTFjxmDHjh2YNGkSvv/+e3Tu3NnYIRVo9erVZSaBoadmz56NzZs3GzuMCuvw4cOYPn06kpOTjR1KqSjR+gkjiomJEQDE33//rXV4mzZtRL169TTKMjIyRFZWlk7LiYiIEEZe1XLlww8/FGZmZiIzM9PYoVQILi4uYuDAgcYOQyY3N1ekp6eL3NxcqezVV18V3t7exguqnNO2TUvK2tpahIaGysqnTZsmAIj79+/rbVmlLSEhQQAQMTExUll2drZIT0/XGA+AiIiIMEhM8+fPFwBEQkKCbFhxfm/KmsLW73nK3SUGpVIJCwsLY4ehk7S0NGOHoJN79+7BysoKlpaWxg6lQrh37x4cHBz0Nr+MjAyo1eoSz8fMzAwqlQpmZuXuMKCznJwcZGVllfpyTGmb6kulSpWgUqn0Nj99Hm/L4++NPpW7vTj/NaHs7GxERkbC398fKpUKTk5OePnll7Fr1y4AQFhYGBYvXgzg6XWtvFeetLQ0jBs3Dp6enlAqlahVqxY+/fRTiHwPuUxPT8fIkSNRpUoV2Nra4rXXXsPNmzehUCg0Ll/kXU87e/YsBgwYgBdeeAEvv/wyAODUqVMICwuDn58fVCoVXF1dMXToUCQlJWksK28e58+fx5tvvgl7e3tUrVoVU6ZMgRAC169fR/fu3WFnZwdXV1d89tlnRdp2OTk5mDlzJqpXrw6lUgkfHx989NFHyMzMlMZRKBSIiYlBWlqatK2evV6YX95loLNnz6Jdu3aoXLkyqlWrhnnz5snGvXfvHoYNGwYXFxeoVCo0bNgQK1eu1Bgn7xrlp59+imXLlkmxNm3aFH///XeR1hMAjh49ii5duuCFF16AtbU1GjRogM8//1xjnN9//x2tWrWCtbU1HBwc0L17d8TFxWmMU5K6yLuEJoTA4sWLZfve5cuX8frrr8PR0RGVK1dGs2bN8Ouvv2osP++a9tq1a/Hxxx+jWrVqqFy5MlJSUhAWFgYbGxtcu3YNXbt2hY2NDapVqybt7//++y/at28Pa2treHt7Y/Xq1VrnnXe9vG3btvj1119x9epVKVYfHx+kpqbC2toao0aNkm3nGzduwNzcHFFRUYXWx9q1axEYGAhbW1vY2dkhICBAVh/JyckYM2YMfHx8oFQq4eHhgcGDB2tcY9d1H4qOjpb2obNnzwIAzp07hz59+sDR0REqlQpNmjTBzz//rDGP5x1XCqKtDYIu35H8FAoF0tLSsHLlSqlO8l8TT05ORlhYGBwcHGBvb48hQ4bgyZMnsnn98MMPCAwMhJWVFRwdHdGvXz9cv379uTEAwM2bNzFs2DC4u7tDqVTC19cX7733npR0PXjwAOPHj0dAQABsbGxgZ2eHkJAQ/PPPP8+dt7Y2CHlWrVqFWrVqQaVSITAwEPv379c6bXGPt9OnT8eECRMAAL6+vtI2vnLlCgDtbRB0+d6uX78es2bNgoeHB1QqFTp06ICLFy8+d5s8fvwYo0ePlr4Lzs7O6NSpE06cOKEx3tGjR9G5c2fY29ujcuXKaNOmDQ4dOlTk9XueSkUaq5Q9evRIa0Ob7Ozs5047ffp0REVF4a233kJQUBBSUlJw7NgxnDhxAp06dcI777yDW7duYdeuXfj+++81phVC4LXXXsMff/yBYcOGoVGjRtixYwcmTJiAmzdvYuHChdK4YWFhWL9+PQYNGoRmzZph3759ePXVVwuM6/XXX4e/vz9mz54tJRu7du3C5cuXMWTIELi6uuLMmTNYtmwZzpw5gz///FP2JXnjjTdQp04dzJkzB7/++is++eQTODo6YunSpWjfvj3mzp2LVatWYfz48WjatClat25d6LZ66623sHLlSvTp0wfjxo3D0aNHERUVhbi4OGzatAkA8P3332PZsmX466+/sHz5cgBAixYtCp3vw4cP0blzZ/Tq1Qt9+/bFhg0b8OGHHyIgIAAhISEAniZYbdu2xcWLFzFixAj4+vrixx9/RFhYGJKTk2U/PqtXr8bjx4/xzjvvQKFQYN68eejVqxcuX7783Ix+165d6Nq1K9zc3DBq1Ci4uroiLi4OW7dulZaze/duhISEwM/PD9OnT0d6ejoWLVqEli1b4sSJE/Dx8SlxXbRu3Rrff/89Bg0ahE6dOmHw4MHS/O7evYsWLVrgyZMnGDlyJJycnLBy5Uq89tpr2LBhA3r27Kmx/JkzZ8LS0hLjx49HZmamdHYnNzcXISEhaN26NebNm4dVq1ZhxIgRsLa2xuTJkzFw4ED06tULS5YsweDBg9G8eXP4+vpq3W6TJ0/Go0ePcOPGDWnft7GxgY2NDXr27Il169ZhwYIFMDc3l6ZZs2YNhBAYOHBgofXRv39/dOjQAXPnzgUAxMXF4dChQ1J9pKamolWrVoiLi8PQoUPRuHFjJCYm4ueff8aNGzdQpUoVnfehmJgYZGRkIDw8HEqlEo6Ojjhz5gxatmyJatWqYeLEibC2tsb69evRo0cPbNy4Udruzzuu6Koo3xFtvv/+eymG8PBwAED16tU1xunbty98fX0RFRWFEydOYPny5XB2dpa2NQDMmjULU6ZMQd++ffHWW2/h/v37WLRoEVq3bo2TJ08Weobr1q1bCAoKQnJyMsLDw1G7dm3cvHkTGzZswJMnT2BpaYnLly9j8+bNeP311+Hr64u7d+9i6dKlaNOmDc6ePQt3d3edt9m+ffuwbt06jBw5EkqlEl999RU6d+6Mv/76S9Y2rbjH2169euH8+fNYs2YNFi5ciCpVqgAAqlatqjUmXb+3c+bMgZmZGcaPH49Hjx5h3rx5GDhwII4ePVrour/77rvYsGEDRowYgbp16yIpKQkHDx5EXFwcGjduDODpPzghISEIDAzEtGnTYGZmhpiYGLRv3x4HDhxAUFCQzusno78rHbrLa4NQ2Ct/GwRvb2+N63ENGzYUr776aqHLKagNwubNmwUA8cknn2iU9+nTRygUCnHx4kUhhBDHjx8XAMTo0aM1xgsLCxMAxLRp06SyvOuC/fv3ly3vyZMnsrI1a9YIAGL//v2yeYSHh0tlOTk5wsPDQygUCjFnzhyp/OHDh8LKykrrNcpnxcbGCgDirbfe0igfP368ACB+//13qSw0NFRYW1sXOr88bdq0EQDEd999J5VlZmYKV1dX0bt3b6ksOjpaABA//PCDVJaVlSWaN28ubGxsREpKihDiv2uUTk5O4sGDB9K4W7ZsEQDEL7/8Umg8OTk5wtfXV3h7e4uHDx9qDFOr1dL7Ro0aCWdnZ5GUlCSV/fPPP8LMzEwMHjxYKtNHXUDL9dTRo0cLAOLAgQNS2ePHj4Wvr6/w8fGRrmH/8ccfAoDw8/OT7T+hoaECgJg9e7YsBoVCIdauXSuVnzt3Trav5s37jz/+kMoKaoOwY8cOAUD89ttvGuUNGjQQbdq0kY3/rFGjRgk7OzuRk5NT4DhTp04VAMRPP/0kG5ZXb7ruQ3Z2duLevXsa8+rQoYMICAgQGRkZGvNv0aKF8Pf3l8qKclzRRts2Lep3pCDPa4MwdOhQjfKePXsKJycn6fOVK1eEubm5mDVrlsZ4//77r6hUqZKsPL/BgwcLMzMzrW3F8uomIyND1u4iISFBKJVKMWPGDI0y5GuDkLcez8o7/h87dkwqu3r1qlCpVKJnz56yaUtyvC3sGn3+3xtdv7d16tTRaMv1+eefCwDi33//lS3rWfb29oW2wVCr1cLf318EBwdrHNeePHkifH19RadOnYq0fs9TJi4xLF68GLt27ZK9GjRo8NxpHRwccObMGVy4cEHn5W7btg3m5uYYOXKkRvm4ceMghMBvv/0GANi+fTsAYPjw4Rrjvf/++wXO+91335WVWVlZSe8zMjKQmJiIZs2aAYDs1BHw9D/+PObm5mjSpAmEEBg2bJhU7uDggFq1auHy5csFxgI8XVcAGDt2rEb5uHHjAEB2ikwXNjY2ePPNN6XPlpaWCAoK0ohp27ZtcHV1Rf/+/aUyCwsLjBw5Eqmpqdi3b5/GPN944w288MIL0udWrVoBwHPX8+TJk0hISMDo0aNl/xXlnaG5ffs2YmNjERYWBkdHR2l4gwYN0KlTJ2lbPUufdQE83R5BQUHS6VDg6XYMDw/HlStXpNPheUJDQzX2n4Jiy4vB2toaffv2lcpr1aoFBweHIsWmTceOHeHu7o5Vq1ZJZadPn8apU6c06l4bBwcHpKWlFXp6fuPGjWjYsKHsPzDgv3rTdR/q3bu3xn9KDx48wO+//46+ffvi8ePHSExMRGJiIpKSkhAcHIwLFy7g5s2bUszFPa5oU5TvSHHlP9a0atUKSUlJSElJAQD89NNPUKvV6Nu3r7TOiYmJcHV1hb+/P/74448C561Wq7F582Z069YNTZo0kQ3PqxulUim1u8jNzUVSUhJsbGxQq1Ytrce2omjevDkCAwOlz15eXujevTt27NiB3NzcQrcBoPvxtih0/d4OGTJEoy1XUY9jDg4OOHr0KG7duqV1eGxsLC5cuIABAwYgKSlJqtO0tDR06NAB+/fv1087pRLPQQ+CgoLQsWNH2evZH4iCzJgxA8nJyahZsyYCAgIwYcIEnDp1qkjLvXr1Ktzd3WFra6tRXqdOHWl43l8zMzPZqdkaNWoUOG9tp3EfPHiAUaNGwcXFBVZWVqhatao03qNHj2Tje3l5aXy2t7eHSqWSThM9W/7w4cMCY3l2HfLH7OrqCgcHB2ldi8PDw0N2eeSFF17QiOnq1avw9/eXNd7Kv63z5F/3vH0hb57p6em4c+eOxgsALl26BACFdo/NW1atWrVkw+rUqSN90QqLpyR1kRdDQct/NsY8BV0WUKlUstOF9vb2WuukqLFpY2ZmhoEDB2Lz5s3S9e1Vq1ZBpVLh9ddfL3Ta4cOHo2bNmggJCYGHhweGDh0qJd15Ll269NwuzbruQ/m32cWLFyGEwJQpU1C1alWN17Rp0wA8beMAlOy4ok1RviPF9bzvyoULFyCEgL+/v2y94+LipHXW5v79+0hJSXlu3ajVaixcuBD+/v5QKpWoUqUKqlatilOnTmk9thWFv7+/rKxmzZp48uQJ7t+/r1Guj+NtUej6vX1e3RRk3rx5OH36NDw9PREUFITp06drJBV5iWtoaKisTpcvX47MzMxir+OzykQbhJJo3bo1Ll26hC1btmDnzp1Yvnw5Fi5ciCVLlmj8Z2Vo2v7b69u3Lw4fPowJEyagUaNGsLGxgVqtRufOnbVme89e6y2sDICsUWVBCmoMVBIljak481y3bh2GDBmit+UVJ57SWO+CFHT2oKAYSiO2wYMHY/78+di8eTP69++P1atXo2vXrrC3ty90OmdnZ8TGxmLHjh347bff8NtvvyEmJgaDBw+WNTDUp/zbLO87Nn78eAQHB2udJi+B1vdxpTT3lefNW61WQ6FQ4LffftM6ro2NTYljmD17NqZMmYKhQ4di5syZcHR0hJmZGUaPHq2X/2SfRx/H29JQ3Hrv27cvWrVqhU2bNmHnzp2YP38+5s6di59++gkhISFS/PPnzy/wRnb6qNdynyAAgKOjI4YMGYIhQ4YgNTUVrVu3xvTp06UvckE/it7e3ti9ezceP36scRbh3Llz0vC8v2q1GgkJCRpZbVFao+Z5+PAh9uzZg8jISEydOlUq19cpzOfJW4cLFy5I2S7wtNFNcnKytK6lufxTp05BrVZr/AeYf1sXVXBwsNZT1nkNuE6fPo2OHTsWGAsAxMfHy4adO3cOVapUgbW1tU7x6Mrb27vA5T8bo6EVlkDWr18fL774IlatWgUPDw9cu3YNixYtKtJ8LS0t0a1bN3Tr1g1qtRrDhw/H0qVLMWXKFNSoUQPVq1fH6dOnC51HSfchPz8/AE8vSxS0bzzreccVQylpUl+9enUIIeDr64uaNWvqNG3VqlVhZ2f33LrZsGED2rVrh2+//VajPDk5WXaWrai0HRvPnz+PypUrP7eRnS7HW122ryG/t25ubhg+fDiGDx+Oe/fuoXHjxpg1axZCQkKk45ydnd1z9+WS7D9l4hJDSeTvImhjY4MaNWpodN3LO9jnv5NUly5dkJubiy+//FKjfOHChVAoFFLr4rz/Nr766iuN8Yp6cAT+yyTzZ46Gumtdly5dtC5vwYIFAFBojwx9Lf/OnTtYt26dVJaTk4NFixbBxsYGbdq00Wl+bm5usktSANC4cWP4+voiOjpaVt95297NzQ2NGjXCypUrNcY5ffo0du7cKW2r0tSlSxf89ddfOHLkiFSWlpaGZcuWwcfHB3Xr1i31GLSxtrYu9NTkoEGDsHPnTkRHR8PJyanQFvh58n9HzczMpPZFed/T3r17459//pF60zwrr95Kug85Ozujbdu2WLp0KW7fvi0b/uxp66IcVwzF2tq6RHf569WrF8zNzREZGSk7/gghZOv6LDMzM/To0QO//PKL1jve5s3P3NxcNu8ff/xRatNRHEeOHNFoK3D9+nVs2bIFr7zySoH/mefR5Xhb0O+DNob43ubm5sq+g87OznB3d5f2v8DAQFSvXh2ffvopUlNTZfN4dl/WZf3yK/dnEOrWrYu2bdsiMDAQjo6OOHbsmNQ9JE9eQ5eRI0ciODgY5ubm6NevH7p164Z27dph8uTJuHLlCho2bIidO3diy5YtGD16tJSlBQYGonfv3oiOjkZSUpLUzfH8+fMAipah2dnZSV3RsrOzUa1aNezcuRMJCQmlsFXkGjZsiNDQUCxbtgzJyclo06YN/vrrL6xcuRI9evRAu3btSnX54eHhWLp0KcLCwnD8+HH4+Phgw4YNOHToEKKjo2XtQIrLzMwMX3/9Nbp164ZGjRphyJAhcHNzw7lz53DmzBns2LEDwNNTcyEhIWjevDmGDRsmdXO0t7c3yG25J06ciDVr1iAkJAQjR46Eo6MjVq5ciYSEBGzcuNFoN9oJDAzEunXrMHbsWDRt2hQ2Njbo1q2bNHzAgAH44IMPsGnTJrz33ntFuonMW2+9hQcPHqB9+/bw8PDA1atXsWjRIjRq1Eg6mzVhwgRs2LABr7/+OoYOHYrAwEA8ePAAP//8M5YsWYKGDRvqZR9avHgxXn75ZQQEBODtt9+Gn58f7t69iyNHjuDGjRtSv/2iHFcMJTAwELt378aCBQvg7u4OX19fvPTSS0Wevnr16vjkk08wadIkXLlyBT169ICtrS0SEhKwadMmhIeHY/z48QVOP3v2bOzcuRNt2rRBeHg46tSpg9u3b+PHH3/EwYMH4eDggK5du2LGjBkYMmQIWrRogX///RerVq2SztoUR/369REcHKzRzREAIiMjnzutLsfbvN+HyZMno1+/frCwsEC3bt20nkU0xPf28ePH8PDwQJ8+fdCwYUPY2Nhg9+7d+Pvvv6X7rJiZmWH58uUICQlBvXr1MGTIEFSrVg03b97EH3/8ATs7O/zyyy86r5+Mzv0e9Kg4t1rO3+3kk08+EUFBQcLBwUFYWVmJ2rVri1mzZmncHjMnJ0e8//77omrVqkKhUGh0qXn8+LEYM2aMcHd3FxYWFsLf31/Mnz9fo+uIEEKkpaWJiIgI4ejoKGxsbESPHj1EfHy8AKDR1a2w25/euHFD9OzZUzg4OAh7e3vx+uuvi1u3bhXYVTL/PArqfqhtO2mTnZ0tIiMjha+vr7CwsBCenp5i0qRJGl2+CluONgUtOzQ0VNZd7u7du2LIkCGiSpUqwtLSUgQEBGh0dxLiv25Q8+fPl80z/3YqzMGDB0WnTp2Era2tsLa2Fg0aNBCLFi3SGGf37t2iZcuWwsrKStjZ2Ylu3bqJs2fPaoyjj7pAAbeNvXTpkujTp49wcHAQKpVKBAUFia1bt2qMk9dd6scff5RNr+v+4O3trdF1T1uXvNTUVDFgwADh4OAgAGjt8tilSxcBQBw+fFg2TJsNGzaIV155RTg7OwtLS0vh5eUl3nnnHXH79m2N8ZKSksSIESNEtWrVhKWlpfDw8BChoaEiMTFRGqek+5AQT7f74MGDhaurq7CwsBDVqlUTXbt2FRs2bJDGKcpxRZuCujkW9Tuizblz50Tr1q2FlZWVACAd/wraN/OOq/m7tW3cuFG8/PLLwtraWlhbW4vatWuLiIgIER8f/9wYrl69KgYPHiyqVq0qlEql8PPzExEREVIXvoyMDDFu3Djh5uYmrKysRMuWLcWRI0dEmzZtNLrB6tLNMSIiQvzwww/C399fKJVK8eKLL2ps18K2gRBFP94KIcTMmTNFtWrVhJmZmca2y/97I0TJvrfa1j+/zMxMMWHCBNGwYUPp+NWwYUPx1VdfycY9efKk6NWrl3BychJKpVJ4e3uLvn37ij179hRp/Z5HIUQptuyq4GJjY/Hiiy/ihx9+KPRGMUQVSc+ePfHvv//q1AaHiMqfct8GwVDS09NlZdHR0TAzM3vuHQyJKorbt2/j119/xaBBg4wdChGVsnLfBsFQ5s2bh+PHj6Ndu3aoVKmS1F0rPDwcnp6exg6PqFQlJCTg0KFDWL58OSwsLPDOO+8YOyQiKmVMEIqoRYsW2LVrF2bOnInU1FR4eXlh+vTpmDx5srFDIyp1+/btw5AhQ+Dl5YWVK1fC1dXV2CERUSljGwQiIiKSYRsEIiIikmGCQERERDJlrg2CWq3GrVu3YGtrWyrPDSAiIqqohBB4/Pgx3N3dS3zjpjKXINy6dYu9AoiIiErg+vXr8PDwKNE8ylyCkHe71OvXr8POzs7I0RAREZUfKSkp8PT01Mvt68tcgpB3WcHOzo4JAhERGV9WFpDXpX3WLMDS0rjxFIE+LtGXuW6OKSkpsLe3x6NHj5ggEBGR8aWlATY2T9+npgKl/Dj4ktDnbyh7MRAREZEMEwQiIiKSKXNtEIpCCIGcnBzk5uYaOxSqICwsLGBubm7sMIiIyoxylyBkZWXh9u3bePLkibFDoQpEoVDAw8MDNnnXGYmITFy5ShDUajUSEhJgbm4Od3d3WFpa8mZKVGJCCNy/fx83btyAv78/zyQQEaGcJQhZWVlQq9Xw9PRE5cqVjR0OVSBVq1bFlStXkJ2dzQSBiAjlLEHIU9LbRxLlxzNRRFQgKyvg9On/3mtx7do1JCYm6jzrKlWqwMvLqyTRlZpymSAQEREZjJkZUK9egYOvXbuGWrXrICNd97ZxKqvKiD8XVyaTBCYIREREJZCYmIiM9Cdw6joOFk5Ff5ZQdtJ1JG39DImJiUwQqOzbu3cv2rVrh4cPH8LBwQErVqzA6NGjkZycbNS42rZti0aNGiE6OtqocRCRCcrKAmbPfvr+o48KvNWyhZMnlK41DBhY6eLFfAO6fv06hg4dKvXA8Pb2xqhRo5CUlGSUeNq2bYvRo0drlLVo0QK3b9+Gvb29UWIiIipzsrOByMinr+xsY0djMEwQDOTy5cto0qQJLly4gDVr1uDixYtYsmQJ9uzZg+bNm+PBgwfGDhEAYGlpCVdXVzbaIyIycRUnQUhLK/iVkVH0cdPTizaujiIiImBpaYmdO3eiTZs28PLyQkhICHbv3o2bN29ict6TwvC0Rf3mzZs1ps873Z/nww8/RM2aNVG5cmX4+flhypQpyH4ms50+fToaNWqE77//Hj4+PrC3t0e/fv3w+PFjAEBYWBj27duHzz//HAqFAgqFAleuXMHevXuhUCgKvaSwZcsWNG7cGCqVCn5+foiMjEROTo7WcXfu3AmVSiWb36hRo9C+fXsAQFJSEvr3749q1aqhcuXKCAgIwJo1awrdnkXZRtevX0ffvn3h4OAAR0dHdO/eHVeuXCl0vkRE9FTFSRBsbAp+9e6tOa6zc8HjhoRojuvjo308HTx48AA7duzA8OHDYZWvi4yrqysGDhyIdevWQZcHa9ra2mLFihU4e/YsPv/8c3zzzTdYuHChxjiXLl3C5s2bsXXrVmzduhX79u3DnDlzAACff/45mjdvjrfffhu3b9/G7du34en5/MY1Bw4cwODBgzFq1CicPXsWS5cuxYoVKzBr1iyt43fo0AEODg7YuHGjVJabm4t169Zh4MCBAICMjAwEBgbi119/xenTpxEeHo5Bgwbhr7/+KvL2yC87OxvBwcGwtbXFgQMHcOjQIdjY2KBz587Iysoq9nyJiExFxUkQyrALFy5ACIE6depoHV6nTh08fPgQ9+/fL/I8P/74Y7Ro0QI+Pj7o1q0bxo8fj/Xr12uMo1arsWLFCtSvXx+tWrXCoEGDsGfPHgCAvb09LC0tUblyZbi6usLV1bVINwiKjIzExIkTERoaCj8/P3Tq1AkzZ87E0qVLtY5vbm6Ofv36YfXq1VLZnj17kJycjN7/n7hVq1YN48ePR6NGjeDn54f3338fnTt3lq2PLtatWwe1Wo3ly5cjICAAderUQUxMDK5du4a9e/cWe75ERKai4vRiSE0teFj+H7579woeN/9NmPR4Svp5ZwgsC2gZq826devwxRdf4NKlS0hNTUVOTo7s2d8+Pj6wtbWVPru5ueFeYeteBP/88w8OHTqkccYgNzcXGRkZePLkidY7XA4cOBDNmjXDrVu34O7ujlWrVuHVV1+Fg4ODNP3s2bOxfv163Lx5E1lZWcjMzCzR3TL/+ecfXLx4UWP9gadnKy5dulTs+RIRmYqKkyBYWxt/3ALUqFEDCoUCcXFx6Nmzp2x4XFwcqlatKv1gKhQKWTLxbPuCI0eOYODAgYiMjERwcDDs7e2xdu1afPbZZxrTWFhYaHxWKBRQq9UlWpfU1FRERkaiV69esmEqlUrrNE2bNkX16tWxdu1avPfee9i0aZNGW4H58+fj888/R3R0NAICAmBtbY3Ro0cXeingedsoNTUVgYGBWLVqlWzaqlWrPm81iYhMXsVJEMowJycndOrUCV999RXGjBmj0Q7hzp07WLVqFSIiIqSyqlWr4vbt29LnCxcuaDy98vDhw/D29tZo2Hj16lWd47K0tNT5kdmNGzdGfHw8atTQra/vwIEDsWrVKnh4eMDMzAyvvvqqNOzQoUPo3r073nzzTQBPL42cP38edevWLXB+z9tGjRs3xrp16+Ds7Cw7s0JEpBOVCshrE1XAP0IVEdsgGMiXX36JzMxMBAcHY//+/bh+/Tq2b9+OTp06oWbNmpg6dao0bvv27fHll1/i5MmTOHbsGN59912NswH+/v64du0a1q5di0uXLuGLL77Apk2bdI7Jx8cHR48exZUrV5CYmFikswtTp07Fd999h8jISJw5cwZxcXFYu3YtPv7440KnGzhwIE6cOIFZs2ahT58+UCqVGuuza9cuHD58GHFxcXjnnXdw9+7dQuf3vG00cOBAVKlSBd27d8eBAweQkJCAvXv3YuTIkbhx48Zz15OISGJuDjRt+vRlQg9zY4JgIP7+/vj777/h5+eHvn37wtvbGyEhIahZs6bUwj7PZ599Bk9PT7Rq1QoDBgzA+PHjNa7Hv/baaxgzZgxGjBiBRo0a4fDhw5gyZYrOMY0fPx7m5uaoW7cuqlatimvXrj13muDgYGzduhU7d+5E06ZN0axZMyxcuBDe3t6FTlejRg0EBQXh1KlTUu+FPB9//DEaN26M4OBgtG3bFq6urujRo0eh83veNqpcuTL2798PLy8v9OrVC3Xq1MGwYcOQkZHBMwpEREWgELr0rTOAlJQU2Nvb49GjR7IDeUZGBhISEuDr61vg9e7yZNq0aViwYAF27dqFZs2aGTsck1bR9i0i0qOsLODzz5++HzVKdqvlEydOIDAwEK6h0TrdajnzzkXcWTkax48fR+PGjfUSamG/obpiGwQjioyMhI+PD/78808EBQXxMdZERGVRdjbwwQdP3w8fXuCzGCoaJghGNmTIEGOHQEREJMN/WYmIiEiGCQIRERHJlMsEoYy1q6QKgPsUEZEmnRKEqKgoNG3aFLa2tnB2dkaPHj0QHx+vMU7btm2lpwPmvd599129BJvXz/3ZG+IQ6UPeXRuL8jwKIiJToFMjxX379iEiIgJNmzZFTk4OPvroI7zyyis4e/YsrJ+5JfHbb7+NGTNmSJ9Lck/9Z5mbm8PBwUF6nkDlypWhUCj0Mm8yXWq1Gvfv30flypVRqRLb7RIRATomCNu3b9f4vGLFCjg7O+P48eNo3bq1VJ73hMDSkDffkj50iOhZZmZm8PLyYsJJRHIqFfDHH/+9NxEl+nfp0aNHAABHR0eN8lWrVuGHH36Aq6srunXrhilTphR4FiEzMxOZmZnS55SUlEKXqVAo4ObmBmdnZ42H8xCVhKWlJe9DQUTamZsDbdsaOwqDK3aCoFarMXr0aLRs2RL169eXygcMGABvb2+4u7vj1KlT+PDDDxEfH4+ffvpJ63yioqIQGRmp8/LNzc15vZiIiKiUFDtBiIiIwOnTp3Hw4EGN8vDwcOl9QEAA3Nzc0KFDB1y6dAnVq1eXzWfSpEkYO3as9DklJQWenp7FDYuIiEi/srOBZcuevg8PB555MFxFVqwEYcSIEdi6dSv2798PDw+PQsd96aWXAAAXL17UmiAolUqNJ/sRERGVKVlZwIgRT9+HhTFB0EYIgffffx+bNm3C3r174evr+9xpYmNjAQBubm7FCpCIiIgMT6cEISIiAqtXr8aWLVtga2uLO3fuAADs7e1hZWWFS5cuYfXq1ejSpQucnJxw6tQpjBkzBq1bt0aDBg1KZQWIiIhI/3RKEL7++msAT2+G9KyYmBiEhYXB0tISu3fvRnR0NNLS0uDp6YnevXvj448/1lvAREREVPp0vsRQGE9PT+zbt69EAREREZHxseM3ERERyTBBICIiIhneeJ6IiKgwSiWwdet/700EEwQiIqLCVKoEvPqqsaMwOF5iICIiIhmeQSAiIipMdjawatXT9wMH8k6KREREhKe3Wh4y5On71183mQSBlxiIiIhIhgkCERERyTBBICIiIhkmCERERCTDBIGIiIhkmCAQERGRDLs5EhERFUapBNav/++9iWCCQEREVJhKlZ7e/8DE8BIDERERyfAMAhERUWFycoBNm56+79nz6RkFE2Aaa0lERFRcmZlA375P36emmkyCwEsMREREJMMEgYiIiGSYIBAREZEMEwQiIiKSYYJAREREMkwQiIiISMY0+moQEREVwbVr15CYmKhZmJ0Nx2nTAAAPTp8GLCw0BsfFxRkqPINigkBERISnyUGt2nWQkf6k4JEiIw0XkJExQSAiIgKQmJiIjPQncOo6DhZOnkWeLv3yMTw68EMpRmYcTBCIiIieYeHkCaVrDemzuToXrRNOAAD2+zZGrpm5xvjZSdcNGp+hMEEgIiIqhGVONmI2PL20UGfMBqRbmj9niopBp14MUVFRaNq0KWxtbeHs7IwePXogPj5eY5yMjAxERETAyckJNjY26N27N+7evavXoImIiKh06ZQg7Nu3DxEREfjzzz+xa9cuZGdn45VXXkFaWpo0zpgxY/DLL7/gxx9/xL59+3Dr1i306tVL74ETERFR6dHpEsP27ds1Pq9YsQLOzs44fvw4WrdujUePHuHbb7/F6tWr0b59ewBATEwM6tSpgz///BPNmjXTX+RERERUakp0o6RHjx4BABwdHQEAx48fR3Z2Njp27CiNU7t2bXh5eeHIkSNa55GZmYmUlBSNFxERERlXsRMEtVqN0aNHo2XLlqhfvz4A4M6dO7C0tISDg4PGuC4uLrhz547W+URFRcHe3l56eXoWvWsJERERlY5iJwgRERE4ffo01q5dW6IAJk2ahEePHkmv69crZncRIiKi8qRY3RxHjBiBrVu3Yv/+/fDw8JDKXV1dkZWVheTkZI2zCHfv3oWrq6vWeSmVSiiVyuKEQUREVOqyzSthSqd3pfemQqczCEIIjBgxAps2bcLvv/8OX19fjeGBgYGwsLDAnj17pLL4+Hhcu3YNzZs310/EREREBpRjXgnfN+6K7xt3RY4JJQg6rWlERARWr16NLVu2wNbWVmpXYG9vDysrK9jb22PYsGEYO3YsHB0dYWdnh/fffx/NmzdnDwYiIqJyRKcE4euvvwYAtG3bVqM8JiYGYWFhAICFCxfCzMwMvXv3RmZmJoKDg/HVV1/pJVgiIiJDM1PnIujGGQDAXx71oDYzjTsp6pQgCCGeO45KpcLixYuxePHiYgdFRERUVihzsrF2zUcAeKtlIiIiMnFMEIiIiEiGCQIRERHJMEEgIiIiGSYIREREJMMEgYiIiGRM55ZQRERExZBjbo7ZbYdI700FEwQiIqJCZJtbYNlLvY0dhsHxEgMRERHJ8AwCERFRIczUuah/9xIA4LRLdd5qmYiIiJ7eavnn78YC4K2WiYiIyMQxQSAiIiIZJghEREQkwwSBiIiIZJggEBERkQwTBCIiIpJhN0ciIqJC5JibI7plf+m9qWCCQEREVIhscwtEvzzQ2GEYHC8xEBERkQzPIBARERVCIdSokXgdAHCxiieEwjT+t2aCQEREVAhVdhZ2/S8CQN6tllVGjsgwTCMNIiIiIp0wQSAiIiIZJghEREQkwwSBiIiIZJggEBERkQwTBCIiIpLROUHYv38/unXrBnd3dygUCmzevFljeFhYGBQKhcarc+fO+oqXiIjIoHLMzbE0qBeWBvXirZYLk5aWhoYNG2Lo0KHo1auX1nE6d+6MmJgY6bNSqSx+hEREREaUbW6BqHZDjR2GwemcIISEhCAkJKTQcZRKJVxdXYsdFBERERlXqbRB2Lt3L5ydnVGrVi289957SEpKKnDczMxMpKSkaLyIiIjKCoVQw+PRXXg8uguFUBs7HIPRe4LQuXNnfPfdd9izZw/mzp2Lffv2ISQkBLm5uVrHj4qKgr29vfTy9PTUd0hERETFpsrOwsElw3BwyTCosrOMHY7B6P1ZDP369ZPeBwQEoEGDBqhevTr27t2LDh06yMafNGkSxo4dK31OSUlhkkBERGRkpd7N0c/PD1WqVMHFixe1DlcqlbCzs9N4ERERkXGVeoJw48YNJCUlwc3NrbQXRURERHqi8yWG1NRUjbMBCQkJiI2NhaOjIxwdHREZGYnevXvD1dUVly5dwgcffIAaNWogODhYr4ETERFR6dE5QTh27BjatWsnfc5rPxAaGoqvv/4ap06dwsqVK5GcnAx3d3e88sormDlzJu+FQEREVI7onCC0bdsWQogCh+/YsaNEAREREZHx6b0XAxERUUWSa2aO7158VXpvKpggEBERFSKrkgWmvvKescMwOD7NkYiIiGR4BoGIiKgwQsAx/eljAB5Y2QEKhZEDMgwmCERERIWwys7EiUUDAQB1xmxAuqXKyBEZBi8xEBERkQwTBCIiIpJhgkBEREQybINQBl27dg2JiYk6T1elShV4eXmVQkRERGRqmCCUMdeuXUOt2nWQkf5E52lVVpURfy6OSQIREZUYE4QyJjExERnpT+DUdRwsnDyLPF120nUkbf0MiYmJTBCIiKjEmCCUURZOnlC61jB2GEREJi/XzBwb6neQ3psKJghERESFyKpkgfGvjjF2GAbHXgxEREQkwzMIREREhRECVtmZAIB0CyVvtUxERFReFae7eFxcnNZyq+xMxC3sA8C0brXMBIGIiCqUknQXp/8wQSAiogqluN3F0y8fw6MDP5RiZOULEwQiIqqQdO0unp10vRSjKX/Yi4GIiIhkmCAQERGRDBMEIiIikmEbBCIiokKozczwa62W0ntTwQSBio2PpSYiU5BZyRIRPSYZOwyDY4JAxcLHUhMRVWxMEKhY+FhqIqKKjQkClQgfS01EFZ1VVoZJ3mrZdFpbEBERUZHpnCDs378f3bp1g7u7OxQKBTZv3qwxXAiBqVOnws3NDVZWVujYsSMuXLigr3iJiIjIAHROENLS0tCwYUMsXrxY6/B58+bhiy++wJIlS3D06FFYW1sjODgYGRkZJQ6WiIiIDEPnNgghISEICQnROkwIgejoaHz88cfo3r07AOC7776Di4sLNm/ejH79+pUsWiIiIjIIvbZBSEhIwJ07d9CxY0epzN7eHi+99BKOHDmidZrMzEykpKRovIiIiMi49Jog3LlzBwDg4uKiUe7i4iINyy8qKgr29vbSy9Oz6F3miIiIqHQYvRfDpEmT8OjRI+l1/Toft0lERGWH2swMv/s1we9+TXir5eJydXUFANy9exdubm5S+d27d9GoUSOt0yiVSiiVSn2GQUREpDeZlSwx9PXpxg7D4PSaCvn6+sLV1RV79uyRylJSUnD06FE0b95cn4siIiKiUqTzGYTU1FRcvHhR+pyQkIDY2Fg4OjrCy8sLo0ePxieffAJ/f3/4+vpiypQpcHd3R48ePfQZNxEREZUinROEY8eOoV27dtLnsWPHAgBCQ0OxYsUKfPDBB0hLS0N4eDiSk5Px8ssvY/v27VCpTOPWlEREVLFYZWXg+JcDAQCBI1aZzK2WdU4Q2rZtCyFEgcMVCgVmzJiBGTNmlCgwIiKisqJydqaxQzA402mOSUREREXGBIGIiIhkmCAQERGRDBMEIiIikmGCQERERDJ6vZMiERFRRaNWKPCnZ33pvalggkBERFSITAsl+g2YY+wwDI6XGIiIiEiGZxAqmLi4OJ2nqVKlCry8vEohmoKVlziJSD+uXbuGxMREnafj9954mCBUELmpDwGFAm+++abO06qsKiP+XJxBvoTlJU4i0p9r166hVu06yEh/ovO0ZeF7b5WVgYNLhgIAXn73f7zVMpUv6sxUQAg4dR0HCyfPIk+XnXQdSVs/Q2JiokG+gOUlTiLSn8TERGSkPynX33un9BSjLt8YmCBUMBZOnlC61jB2GM9VXuIkIv3h9758YSNFIiIikmGCQERERDJMEIiIiEiGCQIRERHJsJEiEZGR8R4BZZtaocA/rv7Se1PBBIGIyIjK+z0CTEGmhRLdQxcaOwyDY4JARGREFeEeAVQxMUEgIioDeI8AKmvYSJGIiKgQquwMHPx6KA5+PRSq7Axjh2MwPINARERUCIUAPFLuSe9NBc8gEBERkQwTBCIiIpLhJYZSVJy+zXFxcaUUDenKFPqmm8I6VnTFPWaUpzrUdR15HNUPJgilpCR9m8n4TKFvuimsY0WWm/oQUCjw5ptvFmv68lCHJV1HKhkmCKWkuH2b0y8fw6MDP5RiZFQUptA33RTWsSJTZ6YCQuhcf0D5qcPiriOPo/rBBKGU6dq3OTvpeilGQ7oyhb7pprCOFZkp1J+xj6NCAZx38pLemwomCERERIXIsFDhlbe+MnYYBqf3XgzTp0+HQqHQeNWuXVvfiyEiIqJSVCpnEOrVq4fdu3f/t5BKPFFBRERUnpTKL3elSpXg6upapHEzMzORmZkpfU5JSSmNkIjKBXY71C9Db09T6NpsCuuYnyo7Az+vHAsAeC10ATIsVEaOyDBKJUG4cOEC3N3doVKp0Lx5c0RFRRX4ZYuKikJkZGRphEFUrrDboX4ZenuaQtdmU1hHbRQCqJl0TXpvKvSeILz00ktYsWIFatWqhdu3byMyMhKtWrXC6dOnYWtrKxt/0qRJGDt2rPQ5JSUFnp66ddkhqgjY7VC/DL09TaFrsymsI/1H7wlCSEiI9L5BgwZ46aWX4O3tjfXr12PYsGGy8ZVKJZRKpb7DICq3TKHbmiEZensau0ueIZjCOpIBnsXg4OCAmjVr4uLFi6W9KCIiItKTUk8QUlNTcenSJbi5uZX2ooiIiEhP9J4gjB8/Hvv27cOVK1dw+PBh9OzZE+bm5ujfv7++F0VERESlRO9tEG7cuIH+/fsjKSkJVatWxcsvv4w///wTVatW1feiiIiISp1QADfsnKX3pkLvCcLatWv1PUu9KG5/aODpvRp0bUhZ3vv9UskUp/5NYZ/hfR6oPMqwUOHl9/5n7DAMziRucVjivrsKM0Co9RsUVUh8PG3BeJ8HovLFJBKE4vbdBf7rv8t+v1QUJXkEb0XfZ3ifB6LyxSQShDzF6Q+d13+X/X5JFyXZ1yo63ueByhtldibWr54IAOg7YA4yLUzj3j0mlSAQERHpykwINLxzQXpvKkr9PghERERU/jBBICIiIhkmCERERCTDNggEQPc++KbQZ58KZ+h9hvsokWExQTBx7LdPujL0PsN9lMg4mCCYuOL226/offapYIbeZ7iPUlmQZGVn7BAMjgkCAeB9Hkh3ht5nuI+SsaRbqhA4crWxwzA4NlIkIiIiGSYIREREJMMEgYiIqBDK7EysXT0Ra1dPhDI709jhGAzbIBARERXCTAg0u35aem8qmCBQuVKcvu2ZmZlQKnV7uEp57EPP+wToF7cnmTomCFQulKgvvMIMEGr9B1VG8D4B+sXtSfQUEwQqF0raF74i96HnfQL0i9uT6CkmCFSuFLcvvCn0oTeFdTQkbk8ydezFQERERDI8g0BERPQcTyx0a+hcETBBICIiKkS6pQp1x240dhgGx0sMREREJMMzCEREJor3eqDCMEEgIjIxvNeDbpQ5Wfh602wAwHs9P0JmJUsjR2QYTBCIiEwM7/WgGzO1Gu0vH5PemwomCEREJor3eqDCsJEiERERyZRagrB48WL4+PhApVLhpZdewl9//VVaiyIiIiI9K5UEYd26dRg7diymTZuGEydOoGHDhggODsa9e/dKY3FERESkZ6WSICxYsABvv/02hgwZgrp162LJkiWoXLky/ve//5XG4oiIiEjP9N5IMSsrC8ePH8ekSZOkMjMzM3Ts2BFHjhyRjZ+ZmYnMzEzp86NHjwAAKSkpeospNTX16bLuXIQ6K0OnafMa5eg6LafjdNzXOF1ZnM4Yyyz30+VkIu8XKf3GGaRXUhZtuuct78ENAE9/o/T1m5c3HyFEyWcm9OzmzZsCgDh8+LBG+YQJE0RQUJBs/GnTpgkAfPHFF1988cWXnl7Xr18v8e+50bs5Tpo0CWPHjpU+q9VqPHjwAE5OTlAoFEaMTH9SUlLg6emJ69evw87Oztjh0P9jvZRNrJeyh3VSNmmrFyEEHj9+DHd39xLPX+8JQpUqVWBubo67d+9qlN+9exeurq6y8ZVKJZRKzdM1Dg4O+g6rTLCzs+OXqwxivZRNrJeyh3VSNuWvF3t7e73MV++NFC0tLREYGIg9e/ZIZWq1Gnv27EHz5s31vTgiIiIqBaVyiWHs2LEIDQ1FkyZNEBQUhOjoaKSlpWHIkCGlsTgiIiLSs1JJEN544w3cv38fU6dOxZ07d9CoUSNs374dLi4upbG4Mk+pVGLatGmySylkXKyXson1UvawTsqm0q4XhRD66AtBREREFQmfxUBEREQyTBCIiIhIhgkCERERyTBBICIiIhkmCHqiy+Otv/nmG7Rq1QovvPACXnjhBXTs2JGPwy4lxX3s+Nq1a6FQKNCjR4/SDdBE6VovycnJiIiIgJubG5RKJWrWrIlt27YZKFrToGudREdHo1atWrCysoKnpyfGjBmDjAzdnglBhdu/fz+6desGd3d3KBQKbN68+bnT7N27F40bN4ZSqUSNGjWwYsWK4gdQ4ps1k1i7dq2wtLQU//vf/8SZM2fE22+/LRwcHMTdu3e1jj9gwACxePFicfLkSREXFyfCwsKEvb29uHHjhoEjr9h0rZc8CQkJolq1aqJVq1aie/fuhgnWhOhaL5mZmaJJkyaiS5cu4uDBgyIhIUHs3btXxMbGGjjyikvXOlm1apVQKpVi1apVIiEhQezYsUO4ubmJMWPGGDjyim3btm1i8uTJ4qeffhIAxKZNmwod//Lly6Jy5cpi7Nix4uzZs2LRokXC3NxcbN++vVjLZ4KgB0FBQSIiIkL6nJubK9zd3UVUVFSRps/JyRG2trZi5cqVpRWiSSpOveTk5IgWLVqI5cuXi9DQUCYIpUDXevn666+Fn5+fyMrKMlSIJkfXOomIiBDt27fXKBs7dqxo2bJlqcZpyoqSIHzwwQeiXr16GmVvvPGGCA4OLtYyeYmhhPIeb92xY0eprLDHW2vz5MkTZGdnw9HRsbTCNDnFrZcZM2bA2dkZw4YNM0SYJqc49fLzzz+jefPmiIiIgIuLC+rXr4/Zs2cjNzfXUGFXaMWpkxYtWuD48ePSZYjLly9j27Zt6NKli0FiJu2OHDmiUY8AEBwcXOTfovyM/jTH8i4xMRG5ubmyu0S6uLjg3LlzRZrHhx9+CHd3d1nFUvEVp14OHjyIb7/9FrGxsQaI0DQVp14uX76M33//HQMHDsS2bdtw8eJFDB8+HNnZ2Zg2bZohwq7QilMnAwYMQGJiIl5++WUIIZCTk4N3330XH330kSFCpgLcuXNHaz2mpKQgPT0dVlZWOs2PZxCMbM6cOVi7di02bdoElUpl7HBM1uPHjzFo0CB88803qFKlirHDoWeo1Wo4Oztj2bJlCAwMxBtvvIHJkydjyZIlxg7NZO3duxezZ8/GV199hRMnTuCnn37Cr7/+ipkzZxo7NNIjnkEoIV0fb/2sTz/9FHPmzMHu3bvRoEGD0gzT5OhaL5cuXcKVK1fQrVs3qUytVgMAKlWqhPj4eFSvXr10gzYBxfm+uLm5wcLCAubm5lJZnTp1cOfOHWRlZcHS0rJUY67oilMnU6ZMwaBBg/DWW28BAAICApCWlobw8HBMnjwZZmb839MYXF1dtdajnZ2dzmcPAJ5BKLHiPt563rx5mDlzJrZv344mTZoYIlSTomu91K5dG//++y9iY2Ol12uvvYZ27dohNjYWnp6ehgy/wirO96Vly5a4ePGilLABwPnz5+Hm5sbkQA+KUydPnjyRJQF5CZzg432Mpnnz5hr1CAC7du0q9LeoUMVq2kga1q5dK5RKpVixYoU4e/asCA8PFw4ODuLOnTtCCCEGDRokJk6cKI0/Z84cYWlpKTZs2CBu374tvR4/fmysVaiQdK2X/NiLoXToWi/Xrl0Ttra2YsSIESI+Pl5s3bpVODs7i08++cRYq1Dh6Fon06ZNE7a2tmLNmjXi8uXLYufOnaJ69eqib9++xlqFCunx48fi5MmT4uTJkwKAWLBggTh58qS4evWqEEKIiRMnikGDBknj53VznDBhgoiLixOLFy9mN8eyYNGiRcLLy0tYWlqKoKAg8eeff0rD2rRpI0JDQ6XP3t7eAoDsNW3aNMMHXsHpUi/5MUEoPbrWy+HDh8VLL70klEql8PPzE7NmzRI5OTkGjrpi06VOsrOzxfTp00X16tWFSqUSnp6eYvjw4eLhw4eGD7wC++OPP7T+VuTVRWhoqGjTpo1smkaNGglLS0vh5+cnYmJiir18Pu6ZiIiIZNgGgYiIiGSYIBAREZEMEwQiIiKSYYJAREREMkwQiIiISIYJAhEREckwQSAiIiIZJghEREQkwwSByMRNnz4dLi4uUCgU2Lx5s0GXfeXKFSgUCj5im6gMYoJABCAsLAwKhQJz5szRKN+8eTMUCoWRoip9cXFxiIyMxNKlS3H79m2EhIQYdPmenp64ffs26tevD+DpY4QVCgWSk5MNGgcRyTFBIPp/KpUKc+fOxcOHD40disFcunQJANC9e3e4urpCqVQWaz7Z2dnFms7c3Byurq6oVKl8P3k+KyvL2CEQ6R0TBKL/17FjR7i6uiIqKqrQ8TZu3Ih69epBqVTCx8cHn332mcZwHx8fzJ49G0OHDoWtrS28vLywbNmy5y7/zJkz6Nq1K+zs7GBra4tWrVpJP+BqtRozZsyAh4cHlEolGjVqhO3bt0vT5p2qX79+PVq1agUrKys0bdoU58+fx99//40mTZrAxsYGISEhuH//PoCnlxa6desGADAzM5POlBR1WevWrUObNm2gUqmwatUqhIWFoUePHpg9ezZcXFzg4OCAGTNmICcnBxMmTICjoyM8PDwQExMjm1dsbCyuXLmCdu3aAQBeeOEFKBQKhIWF4bvvvoOTkxMyMzM1tlePHj0waNAgrdsyKysLI0aMgJubG1QqFby9vTXqNTk5Ge+88w5cXFygUqlQv359bN26Vac6njlzJgYPHgw7OzuEh4cDAA4ePChtf09PT4wcORJpaWnPrXuiMqnYj3kiqkDyntz4008/CZVKJa5fvy6EEGLTpk3i2a/JsWPHhJmZmZgxY4aIj48XMTExwsrKSuOJad7e3sLR0VEsXrxYXLhwQURFRQkzMzNx7ty5Apd/48YN4ejoKHr16iX+/vtvER8fL/73v/9J0yxYsEDY2dmJNWvWiHPnzokPPvhAWFhYiPPnzwshhEhISBAARO3atcX27dvF2bNnRbNmzURgYKBo27atOHjwoDhx4oSoUaOGePfdd4UQTx8lGxMTIwBIjxzXZVk+Pj5i48aN4vLly+LWrVsiNDRU2NraioiICHHu3Dnx7bffCgAiODhYzJo1S5w/f17MnDlTWFhYSNs3b14nT54UOTk5YuPGjQKAiI+PF7dv3xbJycniyZMnwt7eXqxfv17aXnfv3hWVKlUSv//+u9btOX/+fOHp6Sn2798vrly5Ig4cOCBWr14thBAiNzdXNGvWTNSrV0/s3LlTXLp0Sfzyyy9i27ZtOtWxnZ2d+PTTT8XFixell7W1tVi4cKE4f/68OHTokHjxxRdFWFjYc/Y+orKJCQKR0Hy0c7NmzcTQoUOFEPIEYcCAAaJTp04a006YMEHUrVtX+uzt7S3efPNN6bNarRbOzs7i66+/LnD5kyZNEr6+viIrK0vrcHd3dzFr1iyNsqZNm4rhw4cLIf77oV2+fLk0fM2aNQKA2LNnj1QWFRUlatWqJX3Ov366LCs6OlpjnNDQUOHt7S1yc3Olslq1aolWrVpJn3NycoS1tbVYs2aNxrxOnjwphPjv8bb5Hxv83nvviZCQEOnzZ599Jvz8/IRardaytYR4//33Rfv27bUO37FjhzAzMxPx8fFapy1qHffo0UNjnGHDhonw8HCNsgMHDggzMzORnp6udVlEZRkvMRDlM3fuXKxcuRJxcXGyYXFxcWjZsqVGWcuWLXHhwgXk5uZKZQ0aNJDeKxQKuLq64t69ewCAkJAQ2NjYwMbGBvXq1QMAxMbGolWrVrCwsJAtMyUlBbdu3dK63PwxPrtcFxcXAEBAQIBGWV4c2uiyrCZNmsimr1evHszM/jusuLi4aCzf3NwcTk5Ohcagzdtvv42dO3fi5s2bAIAVK1ZIDUu1CQsLQ2xsLGrVqoWRI0di586d0rDY2Fh4eHigZs2aWqctah3nX/9//vkHK1askOrWxsYGwcHBUKvVSEhI0Gl9icqC8t0yiKgUtG7dGsHBwZg0aRLCwsKKNY/8P/QKhQJqtRoAsHz5cqSnp2uMZ2VlVfyAC1hu3o9n/rK8OErK2tq60OXnLa+wbVFUL774Iho2bIjvvvsOr7zyCs6cOYNff/21wPEbN26MhIQE/Pbbb9i9ezf69u2Ljh07YsOGDXrb1vnXPzU1Fe+88w5GjhwpG9fLy0svyyQyJCYIRFrMmTMHjRo1Qq1atTTK69Spg0OHDmmUHTp0CDVr1oS5uXmR5l2tWjVZWYMGDbBy5UpkZ2fLflDt7Ozg7u6OQ4cOoU2bNhrLDQoKKuoqFYkhl6WNpaUlAGj8p57nrbfeQnR0NG7evImOHTvC09Oz0HnZ2dnhjTfewBtvvIE+ffqgc+fOePDgARo0aIAbN27g/PnzWs8iFLeOGzdujLNnz6JGjRpFWVWiMo+XGIi0CAgIwMCBA/HFF19olI8bNw579uzBzJkzcf78eaxcuRJffvklxo8fX6LljRgxAikpKejXrx+OHTuGCxcu4Pvvv0d8fDwAYMKECZg7dy7WrVuH+Ph4TJw4EbGxsRg1alSJlquNIZeVn7e3NxQKBbZu3Yr79+8jNTVVGjZgwADcuHED33zzDYYOHVrofBYsWIA1a9bg3LlzOH/+PH788Ue4urrCwcEBbdq0QevWrdG7d2/s2rVLOtOQ11OjuHX84Ycf4vDhwxgxYgRiY2Nx4cIFbNmyBSNGjCj5hiEyAiYIRAWYMWOG7FR448aNsX79eqxduxb169fH1KlTMWPGjGJfisjj5OSE33//HampqWjTpg0CAwPxzTffSGcTRo4cibFjx2LcuHEICAjA9u3b8fPPP8Pf379Ey9XGkMvKr1q1aoiMjMTEiRPh4uKi8eNqb2+P3r17w8bGBj169Ch0Pra2tpg3bx6aNGmCpk2b4sqVK9i2bZvUPmLjxo1o2rQp+vfvj7p16+KDDz6QzloUt44bNGiAffv24fz582jVqhVefPFFTJ06Fe7u7iXaJkTGohBCCGMHQURUFB06dEC9evVkZ3aISP+YIBBRmffw4UPs3bsXffr0wdmzZ2VtQ4hI/9hIkYjKvBdffBEPHz7E3LlzmRwQGQjPIBAREZEMGykSERGRDBMEIiIikmGCQERERDJMEIiIiEiGCQIRERHJMEEgIiIiGSYIREREJMMEgYiIiGT+D/BulQAiTZf4AAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "A good predictor should have low non-comformity scores, concentrated at the left side of the figure\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "You may see there is a number of data with large non-comformity scores, which means the prediction is badly wrong. This is due to the inherent bias in the GPT3.5 model.\n",
        "\n",
        "We also applied temperature scaling=5 to smooth the probability distribution a bit."
      ],
      "metadata": {
        "id": "etsYbIxyWXU7"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## **Prediction at Test Time and Triggering Human Help**\n",
        "With $\\widehat{q}$, we can construct the predictions sets for test scenarios and observe the empirical coverage (ratio of true option included in the sets) and the average set size.\n",
        "\n",
        "When the set is not a singleton (i.e., has more than one option), we deem the LLM is uncertain about the correct option and **triggers human help**. Below we simulate human always choosing the correct option from the prediction set if there is one. This translates the coverage guarantee to task completion guarantee!"
      ],
      "metadata": {
        "id": "B8874YbMc4O1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@title\n",
        "num_coverage = 0\n",
        "num_help = 0\n",
        "num_success = 0\n",
        "set_size_all = []\n",
        "for data in test_set:\n",
        "  top_logprobs = data['top_logprobs']\n",
        "  top_tokens = data['top_tokens']\n",
        "  true_options = data['true_options']\n",
        "\n",
        "  # normalize the five scores to sum of 1\n",
        "  mc_smx_all = temperature_scaling(top_logprobs, temperature=5)\n",
        "\n",
        "  # include all options with score >= 1-qhat\n",
        "  prediction_set = [\n",
        "            token for token_ind, token in enumerate(top_tokens)\n",
        "            if mc_smx_all[token_ind] >= 1 - qhat\n",
        "        ]\n",
        "  set_size_all.append(len(prediction_set))\n",
        "\n",
        "  # check coverage\n",
        "  flag_coverage = not set(prediction_set).isdisjoint(true_options)\n",
        "  num_coverage += flag_coverage\n",
        "\n",
        "  # check help - if prediction set is not singleton, or set include option E\n",
        "  flag_help = len(prediction_set) != 1 or data['add_mc_prefix'] in prediction_set\n",
        "  num_help += flag_help\n",
        "\n",
        "  # check success - same as coverage\n",
        "  num_success += flag_coverage\n",
        "\n",
        "# get average rate\n",
        "coverage_rate = num_coverage / num_test_data\n",
        "help_rate = num_help / num_test_data\n",
        "success_rate = num_success / num_test_data\n",
        "avg_prediction_set_size = np.mean(set_size_all)\n",
        "\n",
        "# show results\n",
        "print('============== Summary ==============')\n",
        "print('Number of calibration data:', num_calibration_data)\n",
        "print('Number of test data:', num_test_data)\n",
        "# print('Quantile value:', qhat)\n",
        "print('Average prediction set size:', avg_prediction_set_size)\n",
        "print('Marginal coverage guarantee:', 1 - epsilon)\n",
        "print('Empirical coverage:', coverage_rate)\n",
        "print('Help rate:', help_rate)\n",
        "print('Success rate:', success_rate)\n",
        "print('\\nSuccess rate should be the same as empirical coverage, and it is close to the marginal coverage guarantee, 1-epsilon\\n')\n",
        "\n",
        "# plot histogram of prediction set size\n",
        "plt.figure(figsize=(6, 3))\n",
        "plt.hist(\n",
        "    set_size_all, bins=np.arange(-1, 6) + 0.5, edgecolor='k',\n",
        "    linewidth=1\n",
        ")\n",
        "ax = plt.gca()\n",
        "ax.locator_params(integer=True)\n",
        "plt.title('Histogram of prediction set size')\n",
        "plt.xlabel('Prediction set size')\n",
        "plt.ylabel('Frequency')\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "height": 524,
          "base_uri": "https://localhost:8080/"
        },
        "id": "6pf8q6srddxh",
        "outputId": "b895fdc5-5589-4c9a-b0f8-a4a02350c358",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "============== Summary ==============\n",
            "Number of calibration data: 200\n",
            "Number of test data: 100\n",
            "Average prediction set size: 2.61\n",
            "Marginal coverage guarantee: 0.8\n",
            "Empirical coverage: 0.77\n",
            "Help rate: 0.83\n",
            "Success rate: 0.77\n",
            "\n",
            "Success rate should be the same as empirical coverage, and it is close to the marginal coverage guarantee, 1-epsilon\n",
            "\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 600x300 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhMAAAE8CAYAAAB6sTNaAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA6A0lEQVR4nO3deVxU9f4/8NeIzACyyQ6xqIC4golL5L4ilhcVs1yuoKSWmAtqXb51VcrCJU27Keq9Bm1m5lpeFXEDzSVBkTRCJQ0UUEFZlRGZz+8PL/Nz2MQZ9Az6ej4e55HzOed8zns+jvHinM85IxNCCBARERFpqYnUBRAREVHjxjBBREREOmGYICIiIp0wTBAREZFOGCaIiIhIJwwTREREpBOGCSIiItIJwwQRERHphGGCiIiIdMIwQY1GixYtEBISInUZz7xly5ahVatWMDAwQKdOnaQup04LFy6ETCbTaGvoz0lISAhatGjRYP09Kzgu9DCGCZJEbGwsZDIZkpKSalzft29fdOjQQefj7N69GwsXLtS5n+fFvn378O6776JHjx6IiYnBJ598InVJT0V2djYWLlyIlJQUqUt5Ip7190fSayp1AUT1lZ6ejiZNHi//7t69G6tXr2agqKeDBw+iSZMm2LBhA+RyudTlaEWbz0l2djYiIyPRokWLamdj/v3vf0OlUjVghU9fXe9PW8/CuFDDYZigRkOhUEhdwmMrLS1Fs2bNpC6j3m7cuAFjY+MnHiTKysogl8sf+4d+fTT058TQ0LBB+3tWcFzoYbzMQY1G1Wvh5eXliIyMhKenJ4yMjGBtbY2ePXsiPj4ewINruqtXrwYAyGQy9VKptLQUc+bMgYuLCxQKBby8vPDpp5+i6hfp3r17FzNmzICNjQ3MzMzwt7/9DdeuXYNMJtM441F5/f7333/H2LFj0bx5c/Ts2RMAkJqaipCQELRq1QpGRkZwcHDApEmTkJ+fr3Gsyj4uXLiA8ePHw8LCAra2tvjnP/8JIQSysrIQGBgIc3NzODg4YPny5fUau/v37+Ojjz6Cu7s7FAoFWrRogf/7v/+DUqlUbyOTyRATE4PS0lL1WMXGxtbaZ+WlqOTkZLz88sswNjZGy5YtsXbtWo3tDh8+DJlMhk2bNuGDDz7ACy+8ABMTExQVFQEATp48iSFDhsDCwgImJibo06cPfvnll2rHO3r0KLp27QojIyO4u7tj3bp1NdZV05yJgoICzJ49Gy1atIBCoYCzszMmTJiAvLw8HD58GF27dgUATJw4sdp7r2luQH0/OzKZDNOnT8eOHTvQoUMHKBQKtG/fHnv37q11XB/2r3/9C+3bt4eJiQmaN2+OLl26YOPGjRrbXLt2DZMmTYK9vb26/y+//FK9/lHvrybFxcWYNWuWerzs7OwwaNAgnD59Wr1N1XHp27evxr+zh5eHj1VQUIBZs2apx87DwwNLlizhWY5GjmcmSFKFhYXIy8ur1l5eXv7IfRcuXIioqCi8+eab6NatG4qKipCUlITTp09j0KBBmDp1KrKzsxEfH49vvvlGY18hBP72t7/h0KFDCA0NRadOnRAXF4d58+bh2rVr+Oyzz9TbhoSEYPPmzfj73/+Ol156CQkJCXjllVdqreu1116Dp6cnPvnkE/UPl/j4ePz555+YOHEiHBwccP78eaxfvx7nz5/HiRMnqk0ifP3119G2bVssXrwY//3vf7Fo0SJYWVlh3bp16N+/P5YsWYLvvvsOc+fORdeuXdG7d+86x+rNN9/EV199hVGjRmHOnDk4efIkoqKikJaWhu3btwMAvvnmG6xfvx6//vor/vOf/wAAXn755Tr7vX37NoYOHYrRo0djzJgx2Lx5M95++23I5XJMmjRJY9uPPvoIcrkcc+fOhVKphFwux8GDBxEQEABfX18sWLAATZo0QUxMDPr3748jR46gW7duAIDffvsNgwcPhq2tLRYuXIj79+9jwYIFsLe3r7M+ACgpKUGvXr2QlpaGSZMmoXPnzsjLy8NPP/2Eq1evom3btvjwww8xf/58TJkyBb169arzvT/OZwd4EIK2bduGadOmwczMDJ9//jmCgoKQmZkJa2vrWuv+97//jRkzZmDUqFGYOXMmysrKkJqaipMnT2Ls2LEAgOvXr+Oll15ShxZbW1vs2bMHoaGhKCoqwqxZsx77/QHAW2+9hS1btmD69Olo164d8vPzcfToUaSlpaFz58417vP+++/jzTff1Gj79ttvERcXBzs7OwDAnTt30KdPH1y7dg1Tp06Fq6srjh07hoiICOTk5GDlypW11kR6ThBJICYmRgCoc2nfvr3GPm5ubiI4OFj92sfHR7zyyit1HicsLEzU9DHfsWOHACAWLVqk0T5q1Cghk8nEpUuXhBBCJCcnCwBi1qxZGtuFhIQIAGLBggXqtgULFggAYsyYMdWOd+fOnWpt33//vQAgEhMTq/UxZcoUddv9+/eFs7OzkMlkYvHixer227dvC2NjY40xqUlKSooAIN58802N9rlz5woA4uDBg+q24OBg0axZszr7q9SnTx8BQCxfvlzdplQqRadOnYSdnZ24d++eEEKIQ4cOCQCiVatWGuOgUqmEp6en8Pf3FyqVSt1+584d0bJlSzFo0CB12/Dhw4WRkZH466+/1G2///67MDAwqPb3W/VzMn/+fAFAbNu2rdp7qDzuqVOnBAARExNTbZvg4GDh5uamfl3fz44QQgAQcrlco+3s2bMCgPjXv/5V7VgPCwwMrPZvoKrQ0FDh6Ogo8vLyNNrfeOMNYWFhoR7vut5fTSwsLERYWFid21Qdl6p++eUXYWhoKCZNmqRu++ijj0SzZs3EhQsXNLb9xz/+IQwMDERmZma96iP9w8scJKnVq1cjPj6+2uLt7f3IfS0tLXH+/HlcvHjxsY+7e/duGBgYYMaMGRrtc+bMgRACe/bsAQD16ehp06ZpbPfOO+/U2vdbb71Vrc3Y2Fj957KyMuTl5eGll14CAI1Tx5Ue/g3PwMAAXbp0gRACoaGh6nZLS0t4eXnhzz//rLUW4MF7BYDw8HCN9jlz5gAA/vvf/9a5f12aNm2KqVOnql/L5XJMnToVN27cQHJyssa2wcHBGuOQkpKCixcvYuzYscjPz0deXh7y8vJQWlqKAQMGIDExESqVChUVFYiLi8Pw4cPh6uqq3r9t27bw9/d/ZI1bt26Fj48PRowYUW1d1TNC9VHfz06lgQMHwt3dXf3a29sb5ubmj/x7s7S0xNWrV3Hq1Kka1wshsHXrVgwbNgxCCPX45eXlwd/fH4WFhTV+turD0tISJ0+eRHZ2tlb75+bmYtSoUejUqRPWrFmjbv/xxx/Rq1cvNG/eXKPegQMHoqKiAomJiVodj6THyxwkqW7duqFLly7V2iv/Z1OXDz/8EIGBgWjdujU6dOiAIUOG4O9//3u9gshff/0FJycnmJmZabS3bdtWvb7yv02aNEHLli01tvPw8Ki176rbAsCtW7cQGRmJTZs24caNGxrrCgsLq23/8A9NALCwsICRkRFsbGyqtVedd1FV5XuoWrODgwMsLS3V71UbTk5O1SaYtm7dGgBw5coVdWACqo9LZQgMDg6utf/CwkIolUrcvXsXnp6e1dZ7eXmpw1JtMjIyEBQUVPcbeQz1/exUqvp3CTz4fN++fbvO47z33nvYv38/unXrBg8PDwwePBhjx45Fjx49AAA3b95EQUEB1q9fj/Xr19fYR9XPWn0tXboUwcHBcHFxga+vL4YOHYoJEyagVatWj9z3/v37GD16NCoqKrBt2zaNCbEXL15EamoqbG1tG7Rekh7DBDVavXv3RkZGBnbu3Il9+/bhP//5Dz777DOsXbu22rXbp+nh374rjR49GseOHcO8efPQqVMnmJqaQqVSYciQITVOPDMwMKhXG4Bqk/5qo81v4Q2p6rhUvu9ly5bVeruiqampxiTRxkjbv7e2bdsiPT0du3btwt69e7F161asWbMG8+fPR2RkpHr8xo8fX2sgq0+wrsno0aPRq1cvbN++Hfv27cOyZcuwZMkSbNu2DQEBAXXuO2/ePBw/fhz79++Hs7OzxjqVSoVBgwbh3XffrXHfyiBKjQ/DBDVqVlZWmDhxIiZOnIiSkhL07t0bCxcuVIeJ2n6Aurm5Yf/+/SguLtb4DfOPP/5Qr6/8r0qlwuXLlzV+M7506VK9a7x9+zYOHDiAyMhIzJ8/X92uzeUZbVS+h4sXL6p/ewYeTN4rKChQv1dtZGdnV7v99cKFCwDwyKcjVp76Nzc3x8CBA2vdztbWFsbGxjWOV3p6+iNrdHd3x7lz5+rc5nGCVn0/Ow2hWbNmeP311/H666/j3r17GDlyJD7++GNERETA1tYWZmZmqKioqHP8AO2CpKOjI6ZNm4Zp06bhxo0b6Ny5Mz7++OM6w8SmTZuwcuVKrFy5En369Km23t3dHSUlJY+slxofzpmgRqvq6X1TU1N4eHho/CZb+UOuoKBAY9uhQ4eioqICX3zxhUb7Z599BplMpv4fZuU1+Yev+wIPbtmrr8rfTKv+Jvq0Zq4PHTq0xuOtWLECAOq8M+VR7t+/r3GL5r1797Bu3TrY2trC19e3zn19fX3h7u6OTz/9FCUlJdXW37x5E8CD8fP398eOHTuQmZmpXp+Wloa4uLhH1hgUFISzZ8+q71p5WOXfSW2fk5rU97Ojq6qfb7lcjnbt2kEIgfLychgYGCAoKAhbt26tMSxVjh/weO+voqKi2qU3Ozs7ODk51XmW6Ny5c3jzzTcxfvx4zJw5s8ZtRo8ejePHj9f491ZQUID79+8/sj7STzwzQY1Wu3bt0LdvX/j6+sLKygpJSUnq29kqVf5AmzFjBvz9/WFgYIA33ngDw4YNQ79+/fD+++/jypUr8PHxwb59+7Bz507MmjVL/Vuzr68vgoKCsHLlSuTn56tvDa387bs+v/GZm5ujd+/eWLp0KcrLy/HCCy9g3759uHz58hMYlep8fHwQHByM9evXo6CgAH369MGvv/6Kr776CsOHD0e/fv207tvJyQlLlizBlStX0Lp1a/zwww9ISUnB+vXrH/lQoyZNmuA///kPAgIC0L59e0ycOBEvvPACrl27hkOHDsHc3Bw///wzACAyMhJ79+5Fr169MG3aNNy/f1/9DIbU1NQ6jzNv3jxs2bIFr732GiZNmgRfX1/cunULP/30E9auXQsfHx+4u7vD0tISa9euhZmZGZo1a4bu3bvXOP+lvp8dXQ0ePBgODg7o0aMH7O3tkZaWhi+++AKvvPKK+ozI4sWLcejQIXTv3h2TJ09Gu3btcOvWLZw+fRr79+/HrVu3AOCx3l9xcTGcnZ0xatQo+Pj4wNTUFPv378epU6fqfK7JxIkTATy4/Pjtt99qrHv55ZfRqlUrzJs3Dz/99BNeffVVhISEwNfXF6Wlpfjtt9+wZcsWXLlypdq8IGokpLqNhJ5vlbeGnjp1qsb1ffr0eeStoYsWLRLdunUTlpaWwtjYWLRp00Z8/PHH6lsShXhwW+U777wjbG1thUwm07iNsLi4WMyePVs4OTkJQ0ND4enpKZYtW6Zxm6IQQpSWloqwsDBhZWUlTE1NxfDhw0V6eroAoHGrZuVtnTdv3qz2fq5evSpGjBghLC0thYWFhXjttddEdnZ2rbeXVu2jtls2axqnmpSXl4vIyEjRsmVLYWhoKFxcXERERIQoKyur13FqUnnspKQk4efnJ4yMjISbm5v44osvNLarvDX0xx9/rLGfM2fOiJEjRwpra2uhUCiEm5ubGD16tDhw4IDGdgkJCcLX11fI5XLRqlUrsXbtWvV4Pazq50QIIfLz88X06dPFCy+8IORyuXB2dhbBwcEat1Tu3LlTtGvXTjRt2lTjNsqaboGs72cHQI23WNZUY1Xr1q0TvXv3Vo+Lu7u7mDdvnigsLNTY7vr16yIsLEy4uLgIQ0ND4eDgIAYMGCDWr1+vsV1t768qpVIp5s2bJ3x8fISZmZlo1qyZ8PHxEWvWrNHYruq4uLm51Xqb98PHKi4uFhEREcLDw0PI5XJhY2MjXn75ZfHpp59q/NulxkUmRD1nbxGRWkpKCl588UV8++23GDdunNTlSKJv377Iy8t75HwEInr2cc4E0SPcvXu3WtvKlSvRpEmTRz55kojoecA5E0SPsHTpUiQnJ6Nfv35o2rQp9uzZgz179mDKlClwcXGRujwiIskxTBA9wssvv4z4+Hh89NFHKCkpgaurKxYuXIj3339f6tKIiPQC50wQERGRTjhngoiIiHTCMEFEREQ6eebnTKhUKmRnZ8PMzEzy7yYgIiJqTIQQKC4uhpOTE5o0qf38wzMfJrKzsznjnoiISAdZWVnVvrjtYc98mKh87GxWVhbMzc0lroaIiKjxKCoqgouLi8aX2tXkmQ8TlZc2zM3NGSaIiIi08KhpApyASURERDphmCAiIiKdMEwQERGRThgmiIiISCcME0RERKQThgkiIiLSCcMEERER6eSZf84Ekb7IzMxEXl6e1GXoHRsbG7i6ukpdBhHpgGGC6CnIzMyEV5u2KLt7R+pS9I6RsQnS/0hjoCBqxBgmiJ6CvLw8lN29A+tX58DQmt8VU6k8Pwv5u5YjLy+PYYKoEZM0TERHRyM6OhpXrlwBALRv3x7z589HQEAAAKBv375ISEjQ2Gfq1KlYu3bt0y6VqEEYWrtA4eAhdRlERA1K0jDh7OyMxYsXw9PTE0IIfPXVVwgMDMSZM2fQvn17AMDkyZPx4YcfqvcxMTGRqlwiIiKqgaRhYtiwYRqvP/74Y0RHR+PEiRPqMGFiYgIHBwcpyiMiIqJ60JtbQysqKrBp0yaUlpbCz89P3f7dd9/BxsYGHTp0QEREBO7cqXsCm1KpRFFRkcZCRERET47kEzB/++03+Pn5oaysDKampti+fTvatWsHABg7dizc3Nzg5OSE1NRUvPfee0hPT8e2bdtq7S8qKgqRkZFPq3wiIqLnnuRhwsvLCykpKSgsLMSWLVsQHByMhIQEtGvXDlOmTFFv17FjRzg6OmLAgAHIyMiAu7t7jf1FREQgPDxc/bqoqAguLpw9T0RE9KRIHibkcjk8PB7Mbvf19cWpU6ewatUqrFu3rtq23bt3BwBcunSp1jChUCigUCieXMFERESkQW/mTFRSqVRQKpU1rktJSQEAODo6PsWKiIiIqC6SnpmIiIhAQEAAXF1dUVxcjI0bN+Lw4cOIi4tDRkYGNm7ciKFDh8La2hqpqamYPXs2evfuDW9vbynLJiIioodIGiZu3LiBCRMmICcnBxYWFvD29kZcXBwGDRqErKws7N+/HytXrkRpaSlcXFwQFBSEDz74QMqSiYiIqApJw8SGDRtqXefi4lLt6ZdERESkf/RuzgQRERE1LgwTREREpBOGCSIiItIJwwQRERHphGGCiIiIdMIwQURERDphmCAiIiKdMEwQERGRThgmiIiISCcME0RERKQThgkiIiLSCcMEERER6YRhgoiIiHTCMEFEREQ6YZggIiIinTBMEBERkU4YJoiIiEgnDBNERESkE4YJIiIi0gnDBBEREelE0jARHR0Nb29vmJubw9zcHH5+ftizZ496fVlZGcLCwmBtbQ1TU1MEBQXh+vXrElZMREREVUkaJpydnbF48WIkJycjKSkJ/fv3R2BgIM6fPw8AmD17Nn7++Wf8+OOPSEhIQHZ2NkaOHCllyURERFRFUykPPmzYMI3XH3/8MaKjo3HixAk4Oztjw4YN2LhxI/r37w8AiImJQdu2bXHixAm89NJLNfapVCqhVCrVr4uKip7cGyAiIiL9mTNRUVGBTZs2obS0FH5+fkhOTkZ5eTkGDhyo3qZNmzZwdXXF8ePHa+0nKioKFhYW6sXFxeVplE9ERPTckjxM/PbbbzA1NYVCocBbb72F7du3o127dsjNzYVcLoelpaXG9vb29sjNza21v4iICBQWFqqXrKysJ/wOiIiInm+SXuYAAC8vL6SkpKCwsBBbtmxBcHAwEhIStO5PoVBAoVA0YIVERERUF8nDhFwuh4eHBwDA19cXp06dwqpVq/D666/j3r17KCgo0Dg7cf36dTg4OEhULREREVUl+WWOqlQqFZRKJXx9fWFoaIgDBw6o16WnpyMzMxN+fn4SVkhEREQPk/TMREREBAICAuDq6ori4mJs3LgRhw8fRlxcHCwsLBAaGorw8HBYWVnB3Nwc77zzDvz8/Gq9k4OIiIiePknDxI0bNzBhwgTk5OTAwsIC3t7eiIuLw6BBgwAAn332GZo0aYKgoCAolUr4+/tjzZo1UpZMREREVUgaJjZs2FDneiMjI6xevRqrV69+ShURERHR49K7ORNERETUuDBMEBERkU4YJoiIiEgnDBNERESkE4YJIiIi0gnDBBEREemEYYKIiIh0wjBBREREOmGYICIiIp0wTBAREZFOGCaIiIhIJwwTREREpBOGCSIiItIJwwQRERHphGGCiIiIdMIwQURERDphmCAiIiKdMEwQERGRThgmiIiISCeShomoqCh07doVZmZmsLOzw/Dhw5Genq6xTd++fSGTyTSWt956S6KKiYiIqCpJw0RCQgLCwsJw4sQJxMfHo7y8HIMHD0ZpaanGdpMnT0ZOTo56Wbp0qUQVExERUVVNpTz43r17NV7HxsbCzs4OycnJ6N27t7rdxMQEDg4OT7s8IiIiqge9mjNRWFgIALCystJo/+6772BjY4MOHTogIiICd+7cqbUPpVKJoqIijYWIiIieHEnPTDxMpVJh1qxZ6NGjBzp06KBuHzt2LNzc3ODk5ITU1FS89957SE9Px7Zt22rsJyoqCpGRkU+rbCIiouee3oSJsLAwnDt3DkePHtVonzJlivrPHTt2hKOjIwYMGICMjAy4u7tX6yciIgLh4eHq10VFRXBxcXlyhRMRET3n9CJMTJ8+Hbt27UJiYiKcnZ3r3LZ79+4AgEuXLtUYJhQKBRQKxROpk4iIiKqTNEwIIfDOO+9g+/btOHz4MFq2bPnIfVJSUgAAjo6OT7g6IiIiqg9Jw0RYWBg2btyInTt3wszMDLm5uQAACwsLGBsbIyMjAxs3bsTQoUNhbW2N1NRUzJ49G71794a3t7eUpRMREdH/aHU3x59//tkgB4+OjkZhYSH69u0LR0dH9fLDDz8AAORyOfbv34/BgwejTZs2mDNnDoKCgvDzzz83yPGJiIhId1qdmfDw8ECfPn0QGhqKUaNGwcjISKuDCyHqXO/i4oKEhASt+iYiIqKnQ6szE6dPn4a3tzfCw8Ph4OCAqVOn4tdff23o2oiIiKgR0CpMdOrUCatWrUJ2dja+/PJL5OTkoGfPnujQoQNWrFiBmzdvNnSdREREpKd0egJm06ZNMXLkSPz4449YsmQJLl26hLlz58LFxQUTJkxATk5OQ9VJREREekqnMJGUlIRp06bB0dERK1aswNy5c5GRkYH4+HhkZ2cjMDCwoeokIiIiPaXVBMwVK1YgJiYG6enpGDp0KL7++msMHToUTZo8yCYtW7ZEbGwsWrRo0ZC1EtEzKi0tTeoS9JKNjQ1cXV2lLoPokbQKE9HR0Zg0aRJCQkJqfXiUnZ0dNmzYoFNxRPRsqyi5DchkGD9+vNSl6CUjYxOk/5HGQEF6T6swcfHixUduI5fLERwcrE33RPScUClLACFg/eocGFrzO3QeVp6fhfxdy5GXl8cwQXpPqzARExMDU1NTvPbaaxrtP/74I+7cucMQQUSPxdDaBQoHD6nLICItaTUBMyoqCjY2NtXa7ezs8Mknn+hcFBERETUeWoWJzMzMGr+Uy83NDZmZmToXRURERI2HVmHCzs4Oqamp1drPnj0La2trnYsiIiKixkOrMDFmzBjMmDEDhw4dQkVFBSoqKnDw4EHMnDkTb7zxRkPXSERERHpMqwmYH330Ea5cuYIBAwagadMHXahUKkyYMIFzJoiIiJ4zWoUJuVyOH374AR999BHOnj0LY2NjdOzYEW5ubg1dHxEREek5rcJEpdatW6N169YNVQsRERE1QlqFiYqKCsTGxuLAgQO4ceMGVCqVxvqDBw82SHFERESk/7QKEzNnzkRsbCxeeeUVdOjQATKZrKHrIiIiokZCqzCxadMmbN68GUOHDm3oeoiIiKiR0erWULlcDg8PPvqWiIiItAwTc+bMwapVqyCEaOh6iIiIqJHRKkwcPXoU3333Hdzd3TFs2DCMHDlSY6mvqKgodO3aFWZmZrCzs8Pw4cORnp6usU1ZWRnCwsJgbW0NU1NTBAUF4fr169qUTURERE+AVmHC0tISI0aMQJ8+fWBjYwMLCwuNpb4SEhIQFhaGEydOID4+HuXl5Rg8eDBKS0vV28yePRs///wzfvzxRyQkJCA7O/uxAgsRERE9WVp/BXlD2Lt3r8br2NhY2NnZITk5Gb1790ZhYSE2bNiAjRs3on///upjt23bFidOnMBLL73UIHUQERGR9rQ6MwEA9+/fx/79+7Fu3ToUFxcDALKzs1FSUqJ1MYWFhQAAKysrAEBycjLKy8sxcOBA9TZt2rSBq6srjh8/XmMfSqUSRUVFGgsRERE9OVqdmfjrr78wZMgQZGZmQqlUYtCgQTAzM8OSJUugVCqxdu3ax+5TpVJh1qxZ6NGjBzp06AAAyM3NhVwuh6Wlpca29vb2yM3NrbGfqKgoREZGPvbxiYiISDtanZmYOXMmunTpgtu3b8PY2FjdPmLECBw4cECrQsLCwnDu3Dls2rRJq/0rRUREoLCwUL1kZWXp1B8RERHVTaszE0eOHMGxY8cgl8s12lu0aIFr1649dn/Tp0/Hrl27kJiYCGdnZ3W7g4MD7t27h4KCAo2zE9evX4eDg0ONfSkUCigUiseugYiIiLSj1ZkJlUqFioqKau1Xr16FmZlZvfsRQmD69OnYvn07Dh48iJYtW2qs9/X1haGhocbZjvT0dGRmZsLPz0+b0omIiKiBaRUmBg8ejJUrV6pfy2QylJSUYMGCBY/1iO2wsDB8++232LhxI8zMzJCbm4vc3FzcvXsXAGBhYYHQ0FCEh4fj0KFDSE5OxsSJE+Hn58c7OYiIiPSEVpc5li9fDn9/f7Rr1w5lZWUYO3YsLl68CBsbG3z//ff17ic6OhoA0LdvX432mJgYhISEAAA+++wzNGnSBEFBQVAqlfD398eaNWu0KZuIiIieAK3ChLOzM86ePYtNmzYhNTUVJSUlCA0Nxbhx4zQmZD5KfR7HbWRkhNWrV2P16tXalEpERERPmFZhAgCaNm2K8ePHN2QtRERE1AhpFSa+/vrrOtdPmDBBq2KIiIio8dEqTMycOVPjdXl5Oe7cuQO5XA4TExOGCSIioueIVndz3L59W2MpKSlBeno6evbs+VgTMImIiKjx0/q7Oary9PTE4sWLq521ICIiomdbg4UJ4MGkzOzs7IbskoiIiPScVnMmfvrpJ43XQgjk5OTgiy++QI8ePRqkMCIiImoctAoTw4cP13gtk8lga2uL/v37Y/ny5Q1RFxERETUSWoUJlUrV0HUQERFRI9WgcyaIiIjo+aPVmYnw8PB6b7tixQptDkGNVGZmJvLy8qQuQ++kpaVJXQI1UvzsVGdjYwNXV1epy6CHaBUmzpw5gzNnzqC8vBxeXl4AgAsXLsDAwACdO3dWbyeTyRqmSmoUMjMz4dWmLcru3pG6FKJGr6LkNiCT8WsLamBkbIL0P9IYKPSIVmFi2LBhMDMzw1dffYXmzZsDePAgq4kTJ6JXr16YM2dOgxZJjUNeXh7K7t6B9atzYGjtInU5euXun0koPPKt1GVQI6JSlgBC8N9TFeX5WcjftRx5eXkME3pE668g37dvnzpIAEDz5s2xaNEiDB48mGHiOWdo7QKFg4fUZeiV8vwsqUugRor/nqgx0GoCZlFREW7evFmt/ebNmyguLta5KCIiImo8tAoTI0aMwMSJE7Ft2zZcvXoVV69exdatWxEaGoqRI0c2dI1ERESkx7S6zLF27VrMnTsXY8eORXl5+YOOmjZFaGgoli1b1qAFEhERkX7TKkyYmJhgzZo1WLZsGTIyMgAA7u7uaNasWYMWR0RERPpPp4dW5eTkICcnB56enmjWrBmEEA1VFxERETUSWoWJ/Px8DBgwAK1bt8bQoUORk5MDAAgNDX2sOzkSExMxbNgwODk5QSaTYceOHRrrQ0JCIJPJNJYhQ4ZoUzIRERE9IVqFidmzZ8PQ0BCZmZkwMTFRt7/++uvYu3dvvfspLS2Fj48PVq9eXes2Q4YMUZ8BycnJwffff69NyURERPSEaDVnYt++fYiLi4Ozs7NGu6enJ/7666969xMQEICAgIA6t1EoFHBwcNCmTCIiInoKtDozUVpaqnFGotKtW7egUCh0Luphhw8fhp2dHby8vPD2228jPz+/zu2VSiWKioo0FiIiInpytAoTvXr1wtdff61+LZPJoFKpsHTpUvTr16/BihsyZAi+/vprHDhwAEuWLEFCQgICAgJQUVFR6z5RUVGwsLBQLy4ufAwtERHRk6TVZY6lS5diwIABSEpKwr179/Duu+/i/PnzuHXrFn755ZcGK+6NN95Q/7ljx47w9vaGu7s7Dh8+jAEDBtS4T0REhMa3mhYVFTFQEBERPUFanZno0KEDLly4gJ49eyIwMBClpaUYOXIkzpw5A3d394auUa1Vq1awsbHBpUuXat1GoVDA3NxcYyEiIqIn57HPTJSXl2PIkCFYu3Yt3n///SdRU62uXr2K/Px8ODo6PtXjEhERUe0eO0wYGhoiNTW1QQ5eUlKicZbh8uXLSElJgZWVFaysrBAZGYmgoCA4ODggIyMD7777Ljw8PODv798gxyciIiLdaXWZY/z48diwYYPOB09KSsKLL76IF198EQAQHh6OF198EfPnz4eBgQFSU1Pxt7/9Da1bt0ZoaCh8fX1x5MiRBr9jhIiIiLSn1QTM+/fv48svv8T+/fvh6+tb7Ts5VqxYUa9++vbtW+cjuOPi4rQpj4iIiJ6ixwoTf/75J1q0aIFz586hc+fOAIALFy5obCOTyRquOiIiItJ7jxUmPD09kZOTg0OHDgF48Pjszz//HPb29k+kOCIiItJ/jzVnouoliT179qC0tLRBCyIiIqLGRaevIOdXjhMREdFjhYnKrwGv2kZERETPr8eaMyGEQEhIiPrWzLKyMrz11lvV7ubYtm1bw1VIREREeu2xwkRwcLDG6/HjxzdoMURERNT4PFaYiImJeVJ1EBERUSOl0wRMIiIiIoYJIiIi0gnDBBEREemEYYKIiIh0wjBBREREOmGYICIiIp0wTBAREZFOGCaIiIhIJwwTREREpBOGCSIiItIJwwQRERHphGGCiIiIdCJpmEhMTMSwYcPg5OQEmUyGHTt2aKwXQmD+/PlwdHSEsbExBg4ciIsXL0pTLBEREdVI0jBRWloKHx8frF69usb1S5cuxeeff461a9fi5MmTaNasGfz9/VFWVvaUKyUiIqLaPNZXkDe0gIAABAQE1LhOCIGVK1figw8+QGBgIADg66+/hr29PXbs2IE33njjaZZKREREtdDbOROXL19Gbm4uBg4cqG6zsLBA9+7dcfz48Vr3UyqVKCoq0liIiIjoydHbMJGbmwsAsLe312i3t7dXr6tJVFQULCws1IuLi8sTrZOIiOh5p7dhQlsREREoLCxUL1lZWVKXRERE9EzT2zDh4OAAALh+/bpG+/Xr19XraqJQKGBubq6xEBER0ZOjt2GiZcuWcHBwwIEDB9RtRUVFOHnyJPz8/CSsjIiIiB4m6d0cJSUluHTpkvr15cuXkZKSAisrK7i6umLWrFlYtGgRPD090bJlS/zzn/+Ek5MThg8fLl3RREREpEHSMJGUlIR+/fqpX4eHhwMAgoODERsbi3fffRelpaWYMmUKCgoK0LNnT+zduxdGRkZSlUxERERVSBom+vbtCyFEretlMhk+/PBDfPjhh0+xKiIiInocejtngoiIiBoHhgkiIiLSCcMEERER6YRhgoiIiHTCMEFEREQ6YZggIiIinTBMEBERkU4YJoiIiEgnDBNERESkE4YJIiIi0omkj9MmIiLSRlpamtQl6CUbGxu4uro+9eMyTBARUaNRUXIbkMkwfvx4qUvRS0bGJkj/I+2pBwqGCSIiajRUyhJACFi/OgeG1i5Sl6NXyvOzkL9rOfLy8hgmiIiIHsXQ2gUKBw+py6D/4QRMIiIi0gnDBBEREemEYYKIiIh0wjBBREREOmGYICIiIp3odZhYuHAhZDKZxtKmTRupyyIiIqKH6P2toe3bt8f+/fvVr5s21fuSiYiInit6/5O5adOmcHBwkLoMIiIiqoVeX+YAgIsXL8LJyQmtWrXCuHHjkJmZWef2SqUSRUVFGgsRERE9OXodJrp3747Y2Fjs3bsX0dHRuHz5Mnr16oXi4uJa94mKioKFhYV6cXHh41aJiIieJL0OEwEBAXjttdfg7e0Nf39/7N69GwUFBdi8eXOt+0RERKCwsFC9ZGVlPcWKiYiInj96P2fiYZaWlmjdujUuXbpU6zYKhQIKheIpVkVERPR80+szE1WVlJQgIyMDjo6OUpdCRERE/6PXYWLu3LlISEjAlStXcOzYMYwYMQIGBgYYM2aM1KURERHR/+j1ZY6rV69izJgxyM/Ph62tLXr27IkTJ07A1tZW6tKIiIjof/Q6TGzatEnqEoiIiOgR9PoyBxEREek/hgkiIiLSCcMEERER6YRhgoiIiHTCMEFEREQ6YZggIiIinTBMEBERkU4YJoiIiEgnDBNERESkE4YJIiIi0gnDBBEREemEYYKIiIh0wjBBREREOmGYICIiIp0wTBAREZFOGCaIiIhIJwwTREREpBOGCSIiItIJwwQRERHphGGCiIiIdNIowsTq1avRokULGBkZoXv37vj111+lLomIiIj+R+/DxA8//IDw8HAsWLAAp0+fho+PD/z9/XHjxg2pSyMiIiI0gjCxYsUKTJ48GRMnTkS7du2wdu1amJiY4Msvv5S6NCIiIgLQVOoC6nLv3j0kJycjIiJC3dakSRMMHDgQx48fr3EfpVIJpVKpfl1YWAgAKCoqatDacnNzkZub26B9Nnbp6ekAAGXuJajulUlcjX4pz88CwLGpiuNSO45NzTgutSu/dRUAUFJS0mA/8yr7EULUvaHQY9euXRMAxLFjxzTa582bJ7p161bjPgsWLBAAuHDhwoULFy4NtGRlZdX581qvz0xoIyIiAuHh4erXKpUKt27dgrW1NWQymYSVPRlFRUVwcXFBVlYWzM3NpS5Hr3BsasZxqR3HpmYcl9o962MjhEBxcTGcnJzq3E6vw4SNjQ0MDAxw/fp1jfbr16/DwcGhxn0UCgUUCoVGm6Wl5ZMqUW+Ym5s/kx/khsCxqRnHpXYcm5pxXGr3LI+NhYXFI7fR6wmYcrkcvr6+OHDggLpNpVLhwIED8PPzk7AyIiIiqqTXZyYAIDw8HMHBwejSpQu6deuGlStXorS0FBMnTpS6NCIiIkIjCBOvv/46bt68ifnz5yM3NxedOnXC3r17YW9vL3VpekGhUGDBggXVLu0Qx6Y2HJfacWxqxnGpHcfmAZkQj7rfg4iIiKh2ej1ngoiIiPQfwwQRERHphGGCiIiIdMIwQURERDphmGjk+PXs1SUmJmLYsGFwcnKCTCbDjh07pC5JL0RFRaFr164wMzODnZ0dhg8frv4+ledZdHQ0vL291Q8d8vPzw549e6QuS+8sXrwYMpkMs2bNkroUyS1cuBAymUxjadOmjdRlSYphohHj17PXrLS0FD4+Pli9erXUpeiVhIQEhIWF4cSJE4iPj0d5eTkGDx6M0tJSqUuTlLOzMxYvXozk5GQkJSWhf//+CAwMxPnz56UuTW+cOnUK69atg7e3t9Sl6I327dsjJydHvRw9elTqkiTFW0Mbse7du6Nr16744osvADx4OqiLiwveeecd/OMf/5C4Ov0gk8mwfft2DB8+XOpS9M7NmzdhZ2eHhIQE9O7dW+py9IqVlRWWLVuG0NBQqUuRXElJCTp37ow1a9Zg0aJF6NSpE1auXCl1WZJauHAhduzYgZSUFKlL0Rs8M9FIVX49+8CBA9Vtj/p6dqKHFRYWAnjwg5MeqKiowKZNm1BaWspH9v9PWFgYXnnlFY3/1xBw8eJFODk5oVWrVhg3bhwyMzOlLklSev8ETKpZXl4eKioqqj0J1N7eHn/88YdEVVFjoVKpMGvWLPTo0QMdOnSQuhzJ/fbbb/Dz80NZWRlMTU2xfft2tGvXTuqyJLdp0yacPn0ap06dkroUvdK9e3fExsbCy8sLOTk5iIyMRK9evXDu3DmYmZlJXZ4kGCaInkNhYWE4d+7cc3+dt5KXlxdSUlJQWFiILVu2IDg4GAkJCc91oMjKysLMmTMRHx8PIyMjqcvRKwEBAeo/e3t7o3v37nBzc8PmzZuf20tjDBONlDZfz04EANOnT8euXbuQmJgIZ2dnqcvRC3K5HB4eHgAAX19fnDp1CqtWrcK6deskrkw6ycnJuHHjBjp37qxuq6ioQGJiIr744gsolUoYGBhIWKH+sLS0ROvWrXHp0iWpS5EM50w0Uvx6dnpcQghMnz4d27dvx8GDB9GyZUupS9JbKpUKSqVS6jIkNWDAAPz2229ISUlRL126dMG4ceOQkpLCIPGQkpISZGRkwNHRUepSJMMzE40Yv569ZiUlJRq/IVy+fBkpKSmwsrKCq6urhJVJKywsDBs3bsTOnTthZmaG3NxcAICFhQWMjY0lrk46ERERCAgIgKurK4qLi7Fx40YcPnwYcXFxUpcmKTMzs2rzaZo1awZra+vnfp7N3LlzMWzYMLi5uSE7OxsLFiyAgYEBxowZI3VpkmGYaMT49ew1S0pKQr9+/dSvw8PDAQDBwcGIjY2VqCrpRUdHAwD69u2r0R4TE4OQkJCnX5CeuHHjBiZMmICcnBxYWFjA29sbcXFxGDRokNSlkZ66evUqxowZg/z8fNja2qJnz544ceIEbG1tpS5NMnzOBBEREemEcyaIiIhIJwwTREREpBOGCSIiItIJwwQRERHphGGCiIiIdMIwQURERDphmCAiIiKdMEwQERGRThgmiJ5DISEhGD58uPp13759MWvWLJ36bIg+GoOqY0dEfJw2kd4ICQnBV199BQAwNDSEq6srJkyYgP/7v/9D06ZP9p/qtm3bYGhoWK9tDx8+jH79+uH27duwtLTUqg+phISEoKCgADt27NC6j1WrVoEPDibSxDBBpEeGDBmCmJgYKJVK7N69G2FhYTA0NERERES1be/duwe5XN4gx7WystKLPhoDCwsLqUsg0ju8zEGkRxQKBRwcHODm5oa3334bAwcOxE8//QTg/59e//jjj+Hk5AQvLy8AQFZWFkaPHg1LS0tYWVkhMDAQV65cUfdZUVGB8PBwWFpawtraGu+++26136yrXqJQKpV477334OLiAoVCAQ8PD2zYsAFXrlxRf4la8+bNIZPJ1F8SVrWP27dvY8KECWjevDlMTEwQEBCAixcvqtfHxsbC0tIScXFxaNu2LUxNTTFkyBDk5OTUOj63b9/GuHHjYGtrC2NjY3h6eiImJka9vq6xWLhwIb766ivs3LkTMpkMMpkMhw8frvE4W7ZsQceOHWFsbAxra2sMHDgQpaWlGn8PAHDlyhV1Xw8vD3+Z2tGjR9GrVy8YGxvDxcUFM2bMUPdF9KxgmCDSY8bGxrh375769YEDB5Ceno74+Hjs2rUL5eXl8Pf3h5mZGY4cOYJffvlF/UO5cr/ly5cjNjYWX375JY4ePYpbt25h+/btdR53woQJ+P777/H5558jLS0N69atg6mpKVxcXLB161YAQHp6OnJycrBq1aoa+wgJCUFSUhJ++uknHD9+HEIIDB06FOXl5ept7ty5g08//RTffPMNEhMTkZmZiblz59Za1z//+U/8/vvv2LNnD9LS0hAdHQ0bGxsAeORYzJ07F6NHj1YHlpycHLz88svVjpGTk4MxY8Zg0qRJSEtLw+HDhzFy5MgaL224uLio+8rJycGZM2dgbW2N3r17AwAyMjIwZMgQBAUFITU1FT/88AOOHj2K6dOn1zn+RI2OICK9EBwcLAIDA4UQQqhUKhEfHy8UCoWYO3euer29vb1QKpXqfb755hvh5eUlVCqVuk2pVApjY2MRFxcnhBDC0dFRLF26VL2+vLxcODs7q48lhBB9+vQRM2fOFEIIkZ6eLgCI+Pj4Gus8dOiQACBu376t0f5wHxcuXBAAxC+//KJen5eXJ4yNjcXmzZuFEELExMQIAOLSpUvqbVavXi3s7e1rHaNhw4aJiRMn1riuPmPx8BjXJjk5WQAQV65cqXF9bX3cvXtXdO/eXbz66quioqJCCCFEaGiomDJlisZ2R44cEU2aNBF3796tsw6ixoRzJoj0yK5du2Bqaory8nKoVCqMHTsWCxcuVK/v2LGjxjyJs2fP4tKlSzAzM9Pop6ysDBkZGSgsLEROTg66d++uXte0aVN06dKl1kmEKSkpMDAwQJ8+fbR+H2lpaWjatKnGca2treHl5YW0tDR1m4mJCdzd3dWvHR0dcePGjVr7ffvttxEUFITTp09j8ODBGD58uPrswqPGor58fHwwYMAAdOzYEf7+/hg8eDBGjRqF5s2b17nfpEmTUFxcjPj4eDRp0kRdU2pqKr777jv1dkIIqFQqXL58GW3btq13XUT6jGGCSI/069cP0dHRkMvlcHJyqnYXR7NmzTRel5SUwNfXV+OHVSVbW1utajA2NtZqP21UvftDJpPVeadEQEAA/vrrL+zevRvx8fEYMGAAwsLC8OmnnzbYWBgYGCA+Ph7Hjh3Dvn378K9//Qvvv/8+Tp48iZYtW9a4z6JFixAXF4dff/1VI8yUlJRg6tSpmDFjRrV9XF1d610Tkb7jnAkiPdKsWTN4eHjA1dW1XreDdu7cGRcvXoSdnR08PDw0FgsLC1hYWMDR0REnT55U73P//n0kJyfX2mfHjh2hUqmQkJBQ4/rKMyMVFRW19tG2bVvcv39f47j5+flIT09Hu3btHvm+6mJra4vg4GB8++23WLlyJdavXw/g0WNRWXtddVeSyWTo0aMHIiMjcebMGcjl8lrnmWzduhUffvghNm/erHGWpbKm33//vVo9Hh4eDXYnDpE+YJggasTGjRsHGxsbBAYG4siRI7h8+TIOHz6MGTNm4OrVqwCAmTNnYvHixdixYwf++OMPTJs2DQUFBbX22aJFCwQHB2PSpEnYsWOHus/NmzcDANzc3CCTybBr1y7cvHkTJSUl1frw9PREYGAgJk+ejKNHj+Ls2bMYP348XnjhBQQGBmr9fufPn4+dO3fi0qVLOH/+PHbt2qW+VFCfsWjRogVSU1ORnp6OvLw8jcmglU6ePIlPPvkESUlJyMzMxLZt23Dz5s0aL0mcO3cOEyZMwHvvvYf27dsjNzcXubm5uHXrFgDgvffew7FjxzB9+nSkpKTg4sWL2LlzJydg0jOHYYKoETMxMUFiYiJcXV0xcuRItG3bFqGhoSgrK4O5uTkAYM6cOfj73/+O4OBg+Pn5wczMDCNGjKiz3+joaIwaNQrTpk1DmzZtMHnyZPXtjC+88AIiIyPxj3/8A/b29rX+YIyJiYGvry9effVV+Pn5QQiB3bt36/RgK7lcjoiICHh7e6N3794wMDDApk2b6j0WkydPhpeXF7p06QJbW1v88ssv1Y5hbm6OxMREDB06FK1bt8YHH3yA5cuXIyAgoNq2SUlJuHPnDhYtWgRHR0f1MnLkSACAt7c3EhIScOHCBfTq1Qsvvvgi5s+fDycnJ63HgEgfyURdFyiJiIiIHoFnJoiIiEgnDBNERESkE4YJIiIi0gnDBBEREemEYYKIiIh0wjBBREREOmGYICIiIp0wTBAREZFOGCaIiIhIJwwTREREpBOGCSIiItLJ/wPu/bKxw0pgggAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "You may notice that the help rate is very high here. From experiments, we tend to fine the GPT3.5 model significantly underperforms PaLM2-L model, largely due to the bias in GPT3.5 model such as favoring option C/D/E over option A/B. We also tend to find such bias dependent on the context, so simply adjusting the logit_bias argument in the API call does not help significantly."
      ],
      "metadata": {
        "id": "PVnUogycXBcE"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## **Specify Your Own Instruction**\n",
        "You can specify your own instruction here, and also the three objects present on the countertop. If you did not run calibration above, you can choose `skip_calibration` here and applies calibration result based on target success level=0.8.\n",
        "\n",
        "Feel free to use these objects: bottled water, bottled tea, orange soda, RedBull, Coke, Pepsi, Sprite, rice chips, jalapeno chips, kettle chips, multigrain chips, apple, orange, energy bar, dirty sponge with food residue, clean sponge, metal bowl, plastic bowl. You can also specify putting the item in the top or bottom drawer, or in the landfill, or recycling, or landfill bin. For example, \"put the kettle chips in the drawer.\" which is ambiguous in the choice of drawer.\n",
        "\n",
        "This block will then show the prediction set for this scenario --- see if the set makes sense!"
      ],
      "metadata": {
        "id": "hLq1cyarUFs3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "instruction = \"Put the bottled water in the bin.\" #@param {type:\"string\"}\n",
        "scene_objects = \"energy bar, bottled water, rice chips\" #@param {type:\"string\"}\n",
        "skip_calibration = False #@param {type:\"boolean\"}\n",
        "if skip_calibration: qhat = 0.928 # based on epsilon=0.2\n",
        "\n",
        "import random\n",
        "def process_mc_raw(mc_raw, add_mc='an option not listed here'):\n",
        "  mc_all = mc_raw.split('\\n')\n",
        "  mc_processed_all = []\n",
        "  for mc in mc_all:\n",
        "      mc = mc.strip()  # sometimes there is leading space\n",
        "      # skip nonsense\n",
        "      if len(mc) < 5 or mc[0] not in [\n",
        "          'a', 'b', 'c', 'd', 'A', 'B', 'C', 'D', '1', '2', '3', '4'\n",
        "      ]:\n",
        "          continue\n",
        "      mc = mc[2:]  # remove a), b), ...\n",
        "      mc = mc.strip().lower().split('.')[0]\n",
        "      mc_processed_all.append(mc)\n",
        "  if len(mc_processed_all) < 4:\n",
        "      raise 'Cannot extract four options from the raw output.'\n",
        "  # Check if any repeated option - use do nothing as substitue\n",
        "  mc_processed_all = list(set(mc_processed_all))\n",
        "  if len(mc_processed_all) < 4:\n",
        "      num_need = 4 - len(mc_processed_all)\n",
        "      for _ in range(num_need):\n",
        "          mc_processed_all.append('do nothing')\n",
        "  prefix_all = ['A) ', 'B) ', 'C) ', 'D) ']\n",
        "  if add_mc is not None:\n",
        "      mc_processed_all.append(add_mc)\n",
        "      prefix_all.append('E) ')\n",
        "  random.shuffle(mc_processed_all)\n",
        "  mc_prompt = ''\n",
        "  for mc_ind, (prefix, mc) in enumerate(zip(prefix_all, mc_processed_all)):\n",
        "      mc_prompt += prefix + mc\n",
        "      if mc_ind < len(mc_processed_all) - 1:\n",
        "          mc_prompt += '\\n'\n",
        "  add_mc_prefix = prefix_all[mc_processed_all.index(add_mc)][0]\n",
        "  return mc_prompt, mc_processed_all, add_mc_prefix\n",
        "\n",
        "# prompt for generating multiple choice\n",
        "demo_mc_gen_prompt = \"\"\"\n",
        "We: You are a robot operating in an office kitchen. You are in front of a counter with two closed drawers, a top one and a bottom one. There is also a landfill bin, a recycling bin, and a compost bin.\n",
        "\n",
        "We: On the counter, there is an orange soda, a Pepsi, and an apple.\n",
        "We: Put that drink in the top drawer.\n",
        "You:\n",
        "A) open the top drawer and put the orange soda in it\n",
        "B) open the bottom drawer and put the Pepsi in it\n",
        "C) open the bottom drawer and put the orange soda in it\n",
        "D) open the top drawer and put the Pepsi in it\n",
        "\n",
        "We: On the counter, there is an energy bar, a banana, and a microwave.\n",
        "We: Put the snack next to the microwave.\n",
        "You:\n",
        "A) pick up the energy bar and put it next to the microwave\n",
        "B) pick up the banana and put it next to the energy bar\n",
        "C) pick up the banana and put it next to the microwave\n",
        "D) pick up the energy bar and put it next to the banana\n",
        "\n",
        "We: On the counter, there is a Coke, a Sprite, and a sponge.\n",
        "We: Can you dispose of the can? It should have expired.\n",
        "You:\n",
        "A) pick up the sponge and put it in the landfill bin\n",
        "B) pick up the Coke and put it in the recycling bin\n",
        "C) pick up the Sprite and put it in the recycling bin\n",
        "D) pick up the Coke and put it in the landfill bin\n",
        "\n",
        "We: On the counter, there is a bottled water, a bag of jalapeno chips, and a bag of rice chips.\n",
        "We: I would like a bag of chips.\n",
        "You:\n",
        "A) pick up the bottled water\n",
        "B) pick up the jalapeno chips\n",
        "C) pick up the kettle chips\n",
        "D) pick up the rice chips\n",
        "\n",
        "We: On the counter, there is {scene_objects}\n",
        "We: {task}\n",
        "You:\n",
        "\"\"\"\n",
        "demo_mc_gen_prompt = demo_mc_gen_prompt.replace('{task}', instruction)\n",
        "demo_mc_gen_prompt = demo_mc_gen_prompt.replace('{scene_objects}', scene_objects)\n",
        "\n",
        "# Generate multiple choices\n",
        "_, demo_mc_gen_raw = lm(demo_mc_gen_prompt, stop_seq=['We:'], logit_bias={})\n",
        "demo_mc_gen_raw = demo_mc_gen_raw.strip()\n",
        "demo_mc_gen_full, demo_mc_gen_all, demo_add_mc_prefix = process_mc_raw(demo_mc_gen_raw)\n",
        "\n",
        "# get the part of the current scenario from the previous prompt\n",
        "demo_cur_scenario_prompt = demo_mc_gen_prompt.split('\\n\\n')[-1].strip()\n",
        "\n",
        "# get new prompt\n",
        "demo_mc_score_background_prompt = \"\"\"\n",
        "You are a robot operating in an office kitchen. You are in front of a counter with two closed drawers, a top one and a middle one. There is also a landfill bin, a recycling bin, and a compost bin.\n",
        "\"\"\".strip()\n",
        "demo_mc_score_prompt = demo_mc_score_background_prompt + '\\n\\n' + demo_cur_scenario_prompt + '\\n' + demo_mc_gen_full\n",
        "demo_mc_score_prompt += \"\\nWe: Which option is correct? Answer with a single capital letter.\"\n",
        "demo_mc_score_prompt += \"\\nYou:\"\n",
        "\n",
        "# scoring\n",
        "mc_score_response, _ = lm(demo_mc_score_prompt, max_tokens=1, logprobs=5)\n",
        "top_logprobs_full = mc_score_response[\"choices\"][0][\"logprobs\"][\"top_logprobs\"][0]\n",
        "top_tokens = [token.strip() for token in top_logprobs_full.keys()]\n",
        "top_logprobs = [value for value in top_logprobs_full.values()]\n",
        "\n",
        "# get prediction set\n",
        "def temperature_scaling(logits, temperature):\n",
        "    logits = np.array(logits)\n",
        "    logits /= temperature\n",
        "\n",
        "    # apply softmax\n",
        "    logits -= logits.max()\n",
        "    logits = logits - np.log(np.sum(np.exp(logits)))\n",
        "    smx = np.exp(logits)\n",
        "    return smx\n",
        "mc_smx_all = temperature_scaling(top_logprobs, temperature=5)\n",
        "\n",
        "# include all options with score >= 1-qhat\n",
        "prediction_set = [\n",
        "          token for token_ind, token in enumerate(top_tokens)\n",
        "          if mc_smx_all[token_ind] >= 1 - qhat\n",
        "      ]\n",
        "\n",
        "# print\n",
        "print('Multiple choices generated:')\n",
        "print(demo_mc_gen_full)\n",
        "print('\\nPrediction set:', prediction_set)\n",
        "if len(prediction_set) == 1:\n",
        "  print('No help needed!')\n",
        "else:\n",
        "  print('Help needed!')"
      ],
      "metadata": {
        "id": "un9GDoSUilg9",
        "outputId": "a38218e3-c57a-4667-83d1-573d7fa3c863",
        "cellView": "form",
        "collapsed": true,
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Multiple choices generated:\n",
            "A) pick up the bottled water and put it in the landfill bin\n",
            "B) pick up the bottled water and put it in the recycling bin\n",
            "C) pick up the energy bar and put it in the landfill bin\n",
            "D) pick up the rice chips and put it in the landfill bin\n",
            "E) an option not listed here\n",
            "\n",
            "Prediction set: ['B']\n",
            "No help needed!\n"
          ]
        }
      ]
    }
  ]
}